# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# +
import requests
import scrapy
import requests
import re

from scrapy.http import TextResponse
# -

# !tree musinsa

# +
# #!rm -rf musinsa
# -

# !scrapy startproject musinsa

# # 1. 무신사 랭킹페이지 URL

musinsa_ranking_page = "https://store.musinsa.com/app/contents/bestranking"
print(musinsa_ranking_page)

# # 2. 데이터 수집

# +
date, page, = "1d", 1 

url = "https://store.musinsa.com/app/contents/bestranking/\
# ?d_cat_cd=&u_cat_cd=&range={}&price=&ex_soldout=&sale_goods=&new_product_yn=&list_kind=&page={}\
# &display_cnt=90&sex=&popup=&sale_rate=&price1=&price2=&chk_new_product_yn=&chk_sale=&chk_soldout=".format(date, page)

req = requests.get(url)
response = TextResponse(req.url, body=req.text, encoding="utf-8")
# -

print(url)

# ### 2-1. 상품 링크 수집

links = response.xpath('//*[@id="searchList"]/li/div/div[2]/p[2]/a/@href').extract()
links = list(map(lambda data:response.urljoin(data), links))
links[0]

# ### 2-2. 랭킹 데이터 수집

# - map 사용

# +
rankings = response.xpath('//*[@id="searchList"]/li/p/text()[1]').extract()
rankings = list(map(lambda data : "".join(data).strip(), rankings))

rankings[0]
# -

# - 정규 표현식 사용

rankings = response.xpath('//*[@id="searchList"]/li/p/text()[1]').extract()
ranking_1 = rankings[0]
ranking = re.findall(('\d\w'), ranking_1)
ranking[0]

rankings = response.xpath('//*[@id="searchList"]/li/p/text()[1]').extract()
rankings[85:90]

ranking = []
for ranking_1 in rankings:
    ranking_2 = re.findall(('\d\w+'), ranking_1)
    ranking.append(ranking_2[0])

ranking[85:90]

# - join 사용

ranking_test = "".join(ranking_1).strip()
ranking_test

# +
ranking = []
for ranking_1 in rankings:
    ranking_2 = "".join(ranking_1).strip()
    ranking.append(ranking_2)

ranking[85:90]
# -

# ### 2-3 세부페이지의 상품정보 데이터 수집

req2 = requests.get(links[4])
response = TextResponse(req2.url, body=req2.text, encoding="utf-8")

# #### 상품명

# +
# 상품명
product_name = response.xpath('//*[@id="page_product_detail"]/div[4]/div[3]/span/span[1]/text()')[0].extract()

product_name

# +
product_name = response.xpath('//*[@id="page_product_detail"]/div[4]/div[3]/span/span[1]/text()').extract() + [" "]

product_name[0]
# -

# #### 브랜드명

# +
# 브랜드명
brand_name = response.xpath('//*[@id="product_order_info"]/div[1]/ul/li[1]/p[2]/strong/a/text()')[0].extract()

brand_name

# +
brand_name = response.xpath('//*[@id="product_order_info"]/div[1]/ul/li[1]/p[2]/strong/a/text()').extract() + [" "]

brand_name[0]
# -

# #### 상품품번

# +
# 상품 품번
product_num = response.xpath('//*[@id="product_order_info"]/div[1]/ul/li[1]/p[2]/strong/text()')[1].extract()

product_num

# +
product_num = response.xpath('//*[@id="product_order_info"]/div[1]/ul/li[1]/p[2]/strong/text()').extract() + [" "]

product_num[1]
# -

# #### 상품의 성별

req2 = requests.get(links[10])
response = TextResponse(req2.url, body=req2.text, encoding="utf-8")

# +
# 성별
gender = response.xpath('//*[@id="product_order_info"]/div[1]/ul/li[2]/p[2]/span/span/text()')[0:2].extract()

gender
# -

gender = response.xpath('//*[@id="product_order_info"]/div[1]/ul/li[2]/p[2]/span/span/text()')[0:2].extract()
gender = "".join(gender).strip()
gender

# #### 좋아요 개수

# +
# 좋아요 갯수
good_num = response.xpath('//*[@id="product_order_info"]/div[1]/ul/li[5]/p[2]/span/text()')[0].extract()

good_num
# -

# #### 구매후기 개수

# +
# 구매후기 갯수
reviw_count = response.xpath('//*[@id="product_order_info"]/div[1]/ul/li[6]/p[2]/a/text()')[0].extract().replace("건", "")

reviw_count
# -

# #### 판매가격 데이터 수집
# - 배너 있는 상품과 배너 없는 상품을 구분해서 적용
# - origin_price : 판매가 / sale_price : 할인가

# +
# 판매가격
try :
    origin_price = response.xpath('//*[@id="goods_price"]/del/text()')[0].extract()
except:
    origin_price = response.xpath('//*[@id="goods_price"]/text()')[0].extract()
    origin_price = re.findall(('\d+\,\d+'), origin_price)
    
origin_price
# -

# #### origin_price 배너 없는 상품

origin_price_1 = response.xpath('//*[@id="goods_price"]/del/text()').extract()
origin_price_1

# #### origin_price 배너 있는 상품

origin_price_2 = response.xpath('//*[@id="goods_price"]/text()')[0].extract()
origin_price_2 = re.findall(('\d+\,\d+'), origin_price_2)
origin_price_2

# #### sale_price 데이터 수집
# - 베너 있는 상품은 html 구조상 경우의 수가 여러가지 여서, "기획전"으로 통일

# +
# 할인가격
try :
    sale_price = response.xpath('//*[@id="sale_price"]/text()')[0].extract()
except :
    sale_price = "기획전" 
    
sale_price
# -

# #### 주요 구매 고객층 데이터 수집

# +
target_name_1 = response.xpath('//*[@id="graph_summary_area"]/strong[1]/em/text()').extract()
target_name_2 = response.xpath('//*[@id="graph_summary_area"]/span[1]/text()').extract()
target_name_3 = response.xpath('//*[@id="graph_summary_area"]/span[2]/text()').extract()

target_name = target_name_1[0] + " , " + target_name_2[0]

print(target_name_1), print(target_name_2), print(target_name_3), print(target_name)
# -

# # 3. 일간 랭킹, 1 페이지의 상품 데이터 수집

# +
date, page, = "1d", 1 

url = "https://store.musinsa.com/app/contents/bestranking/\
# ?d_cat_cd=&u_cat_cd=&range={}&price=&ex_soldout=&sale_goods=&new_product_yn=&list_kind=&page={}\
# &display_cnt=90&sex=&popup=&sale_rate=&price1=&price2=&chk_new_product_yn=&chk_sale=&chk_soldout=".format(date, page)

req = requests.get(url)
response = TextResponse(req.url, body=req.text, encoding="utf-8")

ranking = response.xpath('//*[@id="searchList"]/li[3]/p/text()').extract()[0]
ranking = re.findall(('\d\w'), ranking)[0]

# 링크 : 정규표현식 대신 map 이 더 간편하다.
# links_1 = response.xpath('//*[@id="searchList"]/li/div[3]/div[2]/p[2]/a/@href').extract()
# links_1 = list(map(lambda data:response.urljoin(data), links_1))
# links_2 = response.xpath('//*[@id="searchList"]/li/div[2]/div[2]/p[2]/a/@href').extract()
# links_2 = list(map(lambda data:response.urljoin(data), links_2))
# links = links_1 + links_2

links = response.xpath('//*[@id="searchList"]/li/div/div[2]/p[2]/a/@href').extract()
links = list(map(lambda data:response.urljoin(data), links))

# 세부페이지
req2 = requests.get(links[2])
response = TextResponse(req2.url, body=req2.text, encoding="utf-8")

product_name = response.xpath('//*[@id="page_product_detail"]/div[4]/div[3]/span/span[1]/text()')[0].extract()
brand_name = response.xpath('//*[@id="product_order_info"]/div[1]/ul/li[1]/p[2]/strong/a/text()')[0].extract()
product_num = response.xpath('//*[@id="product_order_info"]/div[1]/ul/li[1]/p[2]/strong/text()')[1].extract()

try:
    gender = response.xpath('//*[@id="product_order_info"]/div[1]/ul/li[2]/p[2]/span/span/text()').extract()
except:
    gender = response.xpath('//*[@id="product_order_info"]/div[1]/ul/li[2]/p[2]/span/span/text()').extract()

good_num = response.xpath('//*[@id="product_order_info"]/div[1]/ul/li[5]/p[2]/span/text()')[0].extract()
reviw_count = response.xpath('//*[@id="product_order_info"]/div[1]/ul/li[6]/p[2]/a/text()')[0].extract()

try :
    origin_price = response.xpath('//*[@id="goods_price"]/del/text()')[0].extract()
except:
    origin_price = response.xpath('//*[@id="goods_price"]/text()')[0].extract()
    origin_price = re.findall(('\d+\,\d+'), origin_price)

try :
    sale_price = response.xpath('//*[@id="sale_price"]/text()')[0].extract()
except :
    sale_price = "기획전"

# 주요 구매고객층
target_name_1 = response.xpath('//*[@id="graph_summary_area"]/strong[1]/em/text()').extract()
target_name_2 = response.xpath('//*[@id="graph_summary_area"]/span[1]/text()').extract()
target_name_3 = response.xpath('//*[@id="graph_summary_area"]/span[2]/text()').extract()
target_name = target_name_1[0] + " , " + target_name_2[0]
# -

ranking, product_name, brand_name, product_num, gender, good_num, reviw_count, origin_price, sale_price, target_name 

# # 4. 일간 랭킹, 1페이지 전체 상품 데이터 파싱
# - 상품수 90개

# ### for문에서 리스트 없애는 방법

ls = [""] + [""]
# ls

ls[0]

# +
date, page, = "1d", 1 

url = "https://store.musinsa.com/app/contents/bestranking/\
# ?d_cat_cd=&u_cat_cd=&range={}&price=&ex_soldout=&sale_goods=&new_product_yn=&list_kind=&page={}\
# &display_cnt=90&sex=&popup=&sale_rate=&price1=&price2=&chk_new_product_yn=&chk_sale=&chk_soldout=".format(date, page)

req = requests.get(url)
response = TextResponse(req.url, body=req.text, encoding="utf-8")

links = response.xpath('//*[@id="searchList"]/li/div/div[2]/p[2]/a/@href').extract()
links = list(map(lambda data:response.urljoin(data), links))

ranking_df = response.xpath('//*[@id="searchList"]/li/p/text()[1]').extract()

ranking = []
for ranking_1 in ranking_df:
    ranking_2 = re.findall(('\d\w+'), ranking_1)
    ranking.append(ranking_2[0])


dfs = []

for idx in range(0, 90):
    
    req2 = requests.get(links[idx])
    response2 = TextResponse(req2.url, body=req2.text, encoding="utf-8")
    
    product_name = response2.xpath('//*[@id="page_product_detail"]/div[4]/div[3]/span/span[1]/text()').extract() + [" "]
    brand_name = response2.xpath('//*[@id="product_order_info"]/div[1]/ul/li[1]/p[2]/strong/a/text()').extract() + [" "]
    product_num = response2.xpath('//*[@id="product_order_info"]/div[1]/ul/li[1]/p[2]/strong/text()').extract()

    try:
        gender = response2.xpath('//*[@id="product_order_info"]/div[1]/ul/li[2]/p[2]/span/span/text()').extract()
    except:
        gender = response2.xpath('//*[@id="product_order_info"]/div[1]/ul/li[2]/p[2]/span/span/text()').extract()

    good_num = response2.xpath('//*[@id="product_order_info"]/div[1]/ul/li[5]/p[2]/span/text()').extract() + [" "]
    review_count = response2.xpath('//*[@id="product_order_info"]/div[1]/ul/li[6]/p[2]/a/text()').extract() + [" "]

    try :
        origin_price = response2.xpath('//*[@id="goods_price"]/del/text()').extract() + [" "]
    except:
        origin_price = "기획전"
        #origin_price = response.xpath('//*[@id="goods_price"]/text()')[0].extract()
        #origin_price = "".join(origin_price).strip()

    try :
        sale_price = response2.xpath('//*[@id="sale_price"]/text()').extract() + [" "]
    except :
        sale_price = "기획전"

    # 타겟 고객층
    try:
        target_name_1 = response2.xpath('//*[@id="graph_summary_area"]/strong[1]/em/text()').extract()
        target_name_2 = response2.xpath('//*[@id="graph_summary_area"]/span[1]/text()').extract()
        target_name_3 = response2.xpath('//*[@id="graph_summary_area"]/span[2]/text()').extract() 
        target_name = target_name_1[0] + " , " + target_name_2[0]
    except :
        target_name = "-"
    
    dfs.append({
        "product_name" : product_name[0],
        "brand_name" : brand_name[0],
        "product_num" : product_num[1],
        "gender" : gender,
        "good_num" : good_num[0],
        "review_count" : review_count[0],
        "origin_price" : origin_price[0],
        "sale_price" : sale_price[0],
        "target_name" : target_name})
# -

len(links), len(ranking_df), len(ranking)

# ### DataFrame으로 변환

# +
import pandas as pd

df = pd.DataFrame(dfs, columns=["ranking", "brand_name", "product_name", "product_num", "gender", "origin_price", "sale_price", "good_num", "review_count", "target_name"])
df["ranking"] = ranking
df.tail()
# -

df.head()

# # 5. 체크 박스의 카테고리 목록과 코드 파싱
# - 랭킹 페이지의 체크 박스 카테고리 목록과 코드를 확인 할 수 있다.
# - 상의 : 001, 아우터 : 002 등..

# ### 5-1. 랭킹 주기 데이터 수집

response = TextResponse(req.url, body=req.text, encoding="utf-8")
selector = '//*[@id="catelist"]/div[1]/dl/dd/ul/li/a'
range_list = response.xpath(selector).extract()

range_list

range_code = re.split(',', range_list[1])[2]
range_code = re.findall('\d\w', range_code)
range_code

range_name = re.split(',', range_list[1])[3]
range_name = re.findall('[가-힣]+', range_name)
range_name

import pandas as pd

# +
code = []
name = []

for idx in range(0, len(range_list)):
    range_code = re.split(',', range_list[idx])[2]
    range_code = re.findall('\w\w', range_code)
    code.append(range_code)
    
    range_name = re.split(',', range_list[idx])[3]
    range_name = re.findall('[가-힣]+', range_name)
    name.append(range_name)
    
range_pairs = [pair for pair in zip(sum(code, []), sum(name, []))]
range_df = pd.DataFrame(range_pairs, columns=["code", "name"])
range_df
# -

# ### 5-2. 랭킹 카테고리 데이터 수집

selector = '//*[@id="catelist"]/div[2]/dl/dd/ul/li/a'
cate_list = response.xpath(selector).extract()

cate_list[0:3]

cate_code = re.split(',', cate_list[1])[2]
cate_code = re.findall('\d+', cate_code)
cate_code

cate_name = re.split(',', cate_list[1])[3]
cate_name = re.findall('[가-힣]+', cate_name)
cate_name

# +
code = []
name = []

for idx in range(0, len(cate_list)):
    cate_code = re.split(',', cate_list[idx])[2]
    cate_code = re.findall('\d+', cate_code)
    code.append(cate_code)
    
    cate_name = re.split(',', cate_list[idx])[3]
    cate_name = re.findall('[가-힣]+', cate_name)
    name.append(cate_name)

cate_pairs = [pair for pair in zip(sum(code, []), sum(name, []))]
cate_df = pd.DataFrame(cate_pairs, columns=["code", "name"])
cate_df[0:10]
# -

# ### 5-3. 랭킹 주기와 랭킹 카테고리 데이터

check_box_list = pd.concat([range_df, cate_df])
check_box_list.reset_index(drop=True, inplace=True)
check_box_list

# # 6. 마지막 페이지 수 데이터 수집 

response = TextResponse(req.url, body=req.text, encoding="utf-8")
last_page_num = response.xpath('//*[@id="product_list"]/div[2]/div[5]/div/div/a[14]/@onclick').extract()
last_page_num = re.split('\s', last_page_num[0])[1]
last_page_num = re.findall('\d+', last_page_num)
last_page = list(map(int, last_page_num))[-1]
last_page = int(last_page / 50)

last_page

# # 7. Items.py 생성

df.columns

# +
# %%writefile musinsa/musinsa/items.py
import scrapy


class MusinsaItem(scrapy.Item):
    ranking = scrapy.Field()
    brand_name = scrapy.Field()
    product_name = scrapy.Field()
    product_num = scrapy.Field()
    product_spec = scrapy.Field()
    gender = scrapy.Field()
    origin_price = scrapy.Field()
    sale_price = scrapy.Field()
    good_num = scrapy.Field()
    review_count = scrapy.Field()
    target_name = scrapy.Field()
    link = scrapy.Field()


# -

# # 8. Spider version_1 생성 
# - 일간 랭킹 첫 페이지를 수집하는 코드

# +
# %%writefile musinsa/musinsa/spiders/spider.py
import scrapy
import re

from musinsa.items import MusinsaItem

class Spider(scrapy.Spider):
    
    name = "MusinsaRanking"
    allow_domain = ["store.musinsa.com"]
    
    # 1d : 일간, page : 1
    start_urls = ["https://store.musinsa.com/app/contents/bestranking/?d_cat_cd=&u_cat_cd=&range=1d&price=&ex_soldout=&sale_goods=&new_product_yn=&list_kind=&page=1&display_cnt=90&sex=&popup=&sale_rate=&price1=&price2=&chk_new_product_yn=&chk_sale=&chk_soldout="]
    
    def parse(self, response):            
        links = response.xpath('//*[@id="searchList"]/li/div/div[2]/p[2]/a/@href').extract()
        links = list(map(lambda data:response.urljoin(data), links))
        for link in links:
            yield scrapy.Request(link, callback=self.get_content)
                        
    def get_content(self, response):
        item = MusinsaItem()
        
        try:
            product_name_1 = response.xpath('//*[@id="page_product_detail"]/div[4]/div[3]/span/span[1]/text()')[0].extract()
            product_name_2 = response.xpath('//*[@id="page_product_detail"]/div[4]/div[3]/span/span[2]/text()')[0].extract()
            product_name_3 = product_name_1 + " " + product_name_2   
            item["product_name"] = product_name_3.replace("|", "")
        except:
            item["product_name"] = "-"
            
        try:
            item["brand_name"] = response.xpath('//*[@id="product_order_info"]/div[1]/ul/li[1]/p[2]/strong/a/text()')[0].extract()
        except:
            item["brand_name"] = "-"
            
        try:
            item["product_num"] = response.xpath('//*[@id="product_order_info"]/div[1]/ul/li[1]/p[2]/strong/text()')[1].extract()
        except:
            item["product_num"] = "-"
        
        try:
            product_spec_1 =response.xpath('//*[@id="page_product_detail"]/div[4]/div[3]/div[2]/p/a[1]/text()')[0].extract()
            product_spec_2 =response.xpath('//*[@id="page_product_detail"]/div[4]/div[3]/div[2]/p/a[2]/text()')[0].extract()
            item["product_spec"] = product_spec_1 + " / " +product_spec_2 
        except:
            item["product_spec"] = "-"
            
        try:
            gender = response.xpath('//*[@id="product_order_info"]/div[1]/ul/li[2]/p[2]/span/span/text()')[0:2].extract()
            item["gender"] = "".join(gender).strip()
        except:
            item["gender"] = "-"
            #gender = response.xpath('//*[@id="product_order_info"]/div[1]/ul/li[2]/p[2]/span[2]/span/text()').extract()
            #item["gender"] = "".join(gender).strip()
        
        try:
            item["good_num"] = response.xpath('//*[@id="product_order_info"]/div[1]/ul/li[5]/p[2]/span/text()')[0].extract()
        except:
            item["good_num"] = "-"
            
        try:
            item["review_count"] = response.xpath('//*[@id="product_order_info"]/div[1]/ul/li[6]/p[2]/a/text()')[0].extract().replace("건", "")
        except:
            item["review_count"] = "-"
            
        try :
            item["origin_price"] = response.xpath('//*[@id="goods_price"]/del/text()')[0].extract()
        except:
            item["origin_price"] = "기획전"
            #origin_price = response.xpath('//*[@id="goods_price"]/text()')[0].extract()
            #item["origin_price"] = "".join(origin_price).strip()
            # item["origin_price"] = re.findall(('\d+\,\d+'), origin_price)
        
        try :
            item["sale_price"] = response.xpath('//*[@id="sale_price"]/text()')[0].extract()
        except :
            item["sale_price"] = "기획전"
        
        # 타겟 고객층
        target_name_1 = response.xpath('//*[@id="graph_summary_area"]/strong[1]/em/text()').extract()
        target_name_2 = response.xpath('//*[@id="graph_summary_area"]/span[1]/text()').extract()
        target_name_3 = response.xpath('//*[@id="graph_summary_area"]/span[2]/text()').extract()
        item["target_name"] = target_name_1[0] + " , " + target_name_2[0]
        
        item["link"] = response.url
        
        yield item


# -

# ### csv 파일 저장

# 파일 삭제
# !rm -rf musinsa/mr_1d_1p.csv

# %%writefile run.sh
# cd musinsa
scrapy crawl MusinsaRanking -o mr_1d_1p.csv

# ### 권한 설정

# !ls -al run.sh

# !chmod +x run.sh

# !ls -al run.sh

# ### settings.py 수정

# !head -n 22 musinsa/musinsa/settings.py | tail -n 2

# !sed -i 's/ROBOTSTXT_OBEY = True/ROBOTSTXT_OBEY = False/' musinsa/musinsa/settings.py

# !head -n 22 musinsa/musinsa/settings.py | tail -n 2

# ### 요청,회신 자동 조정 (AUTOTHROTTLE) 기능 설정
# - 스크래피의 요청과 회신 과정을 자동으로 정리하여 디버깅 되는 것을 막아준다.

# !tail -n 6 musinsa/musinsa/settings.py

# ### run.sh 실행

# +
# # !./run.sh
# -

# ### csv 파일 불러와서 데이터 프레임 변환

# !ls musinsa/

# +
import pandas as pd

# files = !ls musinsa/
files
# -

"musinsa/{}".format(files[0])

musinsa_df = pd.read_csv("musinsa/{}".format(files[0]))
musinsa_df = musinsa_df[[
    "ranking", "product_spec", "brand_name", "product_name", "product_num", 
    "gender", "origin_price", "sale_price", "good_num", "review_count", "target_name", "link"]]

musinsa_df.tail()

# ### 상품을 30개씩 나눠서 데이터 수집하는 코드 테스트
# - 스크래피 실행시 요청과 응답사이에 디버깅 발생
# - url을 분할해서 수집
# - 실제 사용해보면 디버깅 문제가 개선되진 않는다.

# +
date, page, = "1d", 1 

url = "https://store.musinsa.com/app/contents/bestranking/\
# ?d_cat_cd=&u_cat_cd=&range={}&price=&ex_soldout=&sale_goods=&new_product_yn=&list_kind=&page={}\
# &display_cnt=90&sex=&popup=&sale_rate=&price1=&price2=&chk_new_product_yn=&chk_sale=&chk_soldout=".format(date, page)

req = requests.get(url)
response = TextResponse(req.url, body=req.text, encoding="utf-8")

links = response.xpath('//*[@id="searchList"]/li/div/div[2]/p[2]/a/@href').extract()
links = list(map(lambda data:response.urljoin(data), links))
len(links), links[0]
# -

req2 = requests.get(links[4])
response = TextResponse(req2.url, body=req2.text, encoding="utf-8")

# +
import time

ids = range(0, 90)

for idx in range(0,len(ids), 30):
    start, end = idx, idx + 30
    print("{}부터 {}까지의 링크 주소".format(start, end))
    time.sleep(5)
    
    for link in links[start:end]:
        req = requests.get(link)
        response = TextResponse(req.url, body=req.text, encoding="utf-8")
        brand_name = response.xpath('//*[@id="product_order_info"]/div[1]/ul/li[1]/p[2]/strong/a/text()')[0].extract() 
        print(brand_name, link)
# -

# ### 8-1. 데이터 베이스(MongoDB) 저장

# +
# %%writefile musinsa/musinsa/mongodb.py
import pymongo

client = pymongo.MongoClient('mongodb://172.16.58.3:27017/')
db = client.mr_1d_1p
collection = db.data
# -

# ### 8-2. 파이프라인 추가
# - 데이터 베이스 저장 및 슬랙 전송

# +
# %%writefile musinsa/musinsa/pipelines.py
import json
import requests
import time

from .mongodb import collection


class MusinsaRankingPipeline(object):
    
    def __init__(self):
        self.webhook_url = "<KEY>"
        self.keyword = "ADIDAS"
    
    def send_msg(self, msg):
        payload = {
            "channel": "#data-science",
            "username": "sainthong",
            "text": msg,
        }       
        requests.post(self.webhook_url, json.dumps(payload))
        time.sleep(1)
    
    def process_item(self, item, spider):
        data = {
            "product_spec": item["product_spec"],
            "brand_name": item["brand_name"],
            "product_name": item["product_name"],
            "product_num": item["product_num"],
            "gender": item["gender"],
            "origin_price": item["origin_price"],
            "sale_price": item["sale_price"],
            "good_num": item["good_num"],
            "review_count": item["review_count"],
            "target_name": item["target_name"],
            "link":item["link"]
        }
        
        collection.insert(data)
        
        if self.keyword in item["brand_name"]:
            self.send_msg("카테고리 : {},     브랜드명 : {},     상품명 : {},     판매가격 : {},     세일가격 : {},     좋아요 : {},     링크 : {}"
                          .format(item["product_spec"], item["brand_name"], item["product_name"], item["origin_price"], item["sale_price"], item["good_num"], item["link"]))
        
        return item


# -

# ### settings.py 설정

# !tail -n 6 musinsa/musinsa/settings.py

# !echo "ITEM_PIPELINES = {" >> musinsa/musinsa/settings.py

# !echo "    'musinsa.pipelines.MusinsaRankingPipeline': 300, " >> musinsa/musinsa/settings.py

# !echo "}" >> musinsa/musinsa/settings.py

# !tail -n 8 musinsa/musinsa/settings.py

# !./run.sh

# ### 8-3.데이터 베이스 저장한 데이터 확인

# +
import pymongo

client = pymongo.MongoClient('mongodb://172.16.58.3:27017/')
db = client.mr_1d_1p
collection = db.data
# -

datas = collection.find({}, {"_id":False})

df = pd.DataFrame(datas)
df.tail()

# # 9. Spider_version 2
# - 주간 랭킹 페이지, 카테고리 코드를 적용하여, 100페이지 데이터 수집
# - 서버 메모리 문제로 상의, 아우터 각각 5페이지 데이터 수집 

check_box_list[3:8]

# +
# %%writefile musinsa/musinsa/spiders/spider.py
import requests
import scrapy
import re
import time

from scrapy.http import TextResponse
from musinsa.items import MusinsaItem

class Spider(scrapy.Spider):
    
    name = "MusinsaRanking"
    allow_domain = ["store.musinsa.com"]
    
    def __init__(self, **kwargs):
        # 1w : 주간, page : 1
        self.base_url = "https://store.musinsa.com/app/contents/bestranking/?d_cat_cd=&u_cat_cd={}".format(kwargs["category"])
        self.start_urls = []
        last_page = self.get_last_page()
        for page in range(last_page + 1):
            self.start_urls.append(self.base_url + "&range=1w&price=&ex_soldout=&sale_goods=&new_product_yn=&list_kind=&page={}&display_cnt=90&sex=&popup=&sale_rate=&price1=&price2=&chk_new_product_yn=&chk_sale=&chk_soldout=".format(page))
        super().__init__(**kwargs)
        
    def get_last_page(self):
        req = requests.get(self.base_url)
        response = TextResponse(req.url, body=req.text, encoding="utf-8")
        last_page_num = response.xpath('//*[@id="product_list"]/div[2]/div[5]/div/div/a[14]/@onclick').extract()
        last_page_num = re.split('\s', last_page_num[0])[1]
        last_page_num = re.findall('\d+', last_page_num)
        last_page = list(map(int, last_page_num))[-1]
        last_page = int(last_page / 20)
        return last_page
        
    def start_requests(self):
        for url in self.start_urls:
            yield scrapy.Request(url=url, callback=self.parse)

    def parse(self, response):
        links = response.xpath('//*[@id="searchList"]/li/div/div[2]/p[2]/a/@href').extract()
        links = list(map(lambda data:response.urljoin(data), links))
        for link in links:
            time.sleep(1)
            yield scrapy.Request(link, callback=self.get_content)
   
    def get_content(self, response):
        item = MusinsaItem()
        try:
            product_name_1 = response.xpath('//*[@id="page_product_detail"]/div[4]/div[3]/span/span[1]/text()')[0].extract()
            product_name_2 = response.xpath('//*[@id="page_product_detail"]/div[4]/div[3]/span/span[2]/text()')[0].extract()
            product_name_3 = product_name_1 + " " + product_name_2   
            item["product_name"] = product_name_3.replace("|", "")
        except:
            item["product_name"] = "-"
            
        try:
            item["brand_name"] = response.xpath('//*[@id="product_order_info"]/div[1]/ul/li[1]/p[2]/strong/a/text()')[0].extract()
        except:
            item["brand_name"] = "-"
            
        try:
            item["product_num"] = response.xpath('//*[@id="product_order_info"]/div[1]/ul/li[1]/p[2]/strong/text()')[1].extract()
        except:
            item["product_num"] = "-"
        
        try:
            product_spec_1 =response.xpath('//*[@id="page_product_detail"]/div[4]/div[3]/div[2]/p/a[1]/text()')[0].extract()
            product_spec_2 =response.xpath('//*[@id="page_product_detail"]/div[4]/div[3]/div[2]/p/a[2]/text()')[0].extract()
            item["product_spec"] = product_spec_1 + " / " +product_spec_2 
        except:
            item["product_spec"] = "-"
            
        try:
            gender = response.xpath('//*[@id="product_order_info"]/div[1]/ul/li[2]/p[2]/span/span/text()')[0:2].extract()
            item["gender"] = "".join(gender).strip()
        except:
            item["gender"] = "-"
            #gender = response.xpath('//*[@id="product_order_info"]/div[1]/ul/li[2]/p[2]/span[2]/span/text()').extract()
            #item["gender"] = "".join(gender).strip()
        
        try:
            item["good_num"] = response.xpath('//*[@id="product_order_info"]/div[1]/ul/li[5]/p[2]/span/text()')[0].extract()
        except:
            item["good_num"] = "-"
            
        try:
            item["review_count"] = response.xpath('//*[@id="product_order_info"]/div[1]/ul/li[6]/p[2]/a/text()')[0].extract().replace("건", "")
        except:
            item["review_count"] = "-"
            
        try :
            item["origin_price"] = response.xpath('//*[@id="goods_price"]/del/text()')[0].extract()
        except:
            #item["origin_price"] = "기획전"
            origin_price = response.xpath('//*[@id="goods_price"]/text()')[0].extract()
            item["origin_price"] = "".join(origin_price).strip()
            #item["origin_price"] = re.findall(('\d+\,\d+'), origin_price)
        
        try :
            item["sale_price"] = response.xpath('//*[@id="sale_price"]/text()')[0].extract()
        except :
            item["sale_price"] = "기획전"
        
        # 타겟 고객층
        target_name_1 = response.xpath('//*[@id="graph_summary_area"]/strong[1]/em/text()').extract()
        target_name_2 = response.xpath('//*[@id="graph_summary_area"]/span[1]/text()').extract()
        target_name_3 = response.xpath('//*[@id="graph_summary_area"]/span[2]/text()').extract()
        item["target_name"] = target_name_1[0] + " , " + target_name_2[0]
        
        item["link"] = response.url
        
        yield item


# -

# ### 9-1. 상의, 아우터 데이터 csv 파일 저장

# %%writefile run.sh
# cd musinsa
scrapy crawl MusinsaRanking -o mr_1w_5p_001.csv -a category=001 #상 의
scrapy crawl MusinsaRanking -o mr_1w_5p_002.csv -a category=002 #아우터

# ### 9-2. 데이터 베이스 저장

# +
# %%writefile musinsa/musinsa/mongodb.py
import pymongo

client = pymongo.MongoClient('mongodb://172.16.58.3:27017/')
db = client.mr_1w_5p_001_002
collection = db.data
# -

# ### 9-3. pipeline 생성
# - 데이터 베이스 저장 및 슬랙 전송
# - "THISISNEVERTHAT" 키워드에 해당하는 상품 데이터 슬랙 전송

# +
# %%writefile musinsa/musinsa/pipelines.py
import json
import requests
from .mongodb import collection


class MusinsaRankingPipeline(object):
    
    def __init__(self):
        self.webhook_url = "<KEY>"
        self.keyword = "THISISNEVERTHAT"
    
    def send_msg(self, msg):
        payload = {
            "channel": "#musinsa_crawling.",
            "username": "sainthong",
            "text": msg,
        }
        
        requests.post(self.webhook_url, json.dumps(payload))
        time.sleep(1)
    
    def process_item(self, item, spider):
        data = {
            "product_spec": item["product_spec"],
            "brand_name": item["brand_name"],
            "product_name": item["product_name"],
            "product_num": item["product_num"],
            "gender": item["gender"],
            "origin_price": item["origin_price"],
            "sale_price": item["sale_price"],
            "good_num": item["good_num"],
            "review_count": item["review_count"],
            "target_name": item["target_name"],
            "link":item["link"]
        }
        
        collection.insert(data)
        
        if self.keyword in item["brand_name"]:
            self.send_msg("카테고리 : {},     브랜드명 : {},     상품명 : {},     판매가격 : {},     세일가격 : {},     좋아요 : {},     링크 : {}"
                          .format(item["product_spec"], item["brand_name"], item["product_name"], item["origin_price"], item["sale_price"], item["good_num"], item["link"]))
        
        return item


# +
# # !./run.sh
# -

# ### 수집한 데이터 확인

categories = ["001", "002"]

# +
import pandas as pd

top_outer_dfs = [pd.read_csv("musinsa/mr_1w_5p_{}.csv".format(category)) for category in categories]
top_outer_dfs
# -

# ### 수집한 데이터의 수 확인

[(category, len(df)) for category, df in zip(categories, top_outer_dfs)]

# ### 상의, 아우터 데이터를 데이터 프레임으로 변환

# file = !ls ./musinsa
file

"musinsa/{}".format(file[1])

# ### 상의 : 001 

top_df = pd.read_csv("musinsa/{}".format(file[1]))
top_df = top_df[[
    "ranking", "product_spec", "brand_name", "product_name", "product_num", 
    "gender", "origin_price", "sale_price", "good_num", "review_count", "target_name", "link"]]
top_df.tail()

# ### 아우터 : 002 

outer_df = pd.read_csv("musinsa/{}".format(file[2]))
outer_df = outer_df[[
    "ranking", "product_spec", "brand_name", "product_name", "product_num", 
    "gender", "origin_price", "sale_price", "good_num", "review_count", "target_name", "link"]]
outer_df.tail()

# ### 상의 와 아우터 데이터 통합

top_outer_df = pd.concat([top_df, outer_df])
top_outer_df.reset_index(drop=True, inplace=True)
top_outer_df.tail()



StarcoderdataJupyterScriptsDedupFiltered