# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     name: python3
# ---

# + [markdown] id="view-in-github" colab_type="text"
# <a href="https://colab.research.google.com/github/lvisdd/nlp_tutorial/blob/master/chiVe.ipynb" target="_parent"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a>

# + [markdown] id="SYJLVtEKMFtY" colab_type="text"
# # [Sudachiベースの学習済みWord2Vecモデルを使う](https://ohke.hateblo.jp/entry/2019/06/01/120000)

# + [markdown] id="5JTYyluZOydf" colab_type="text"
# ## [chiVe: Japanese Word Embedding with Sudachi & NWJC](https://github.com/WorksApplications/chiVe)

# + id="7Ko40Kw-Lysx" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 208} outputId="664f30d1-ccae-46c4-d411-b738964e4b69"
# !wget https://object-storage.tyo2.conoha.io/v1/nc_2520839e1f9641b08211a5c85243124a/chive/chive-1.0-mc5-20190314.tar.gz

# + id="1UGAFuFpMSSP" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 69} outputId="5fa1388e-338e-4de7-83e9-616be7075645"
# !tar -zxvf ./chive-1.0-mc5-20190314.tar.gz

# + id="Dn3HjMx-MfbP" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 541} outputId="1430d236-2bb2-4fa6-fec7-6de00eb11f20"
# !pip install gensim

# + id="wsP3qO0MMhIf" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 89} outputId="3eb79a29-6259-4b0c-f67c-4b5752200720"
# モデルのロード
from gensim.models import KeyedVectors
from gensim.test.utils import datapath

nwjc_model = KeyedVectors.load_word2vec_format(
    datapath('/content/chive-1.0-mc5-20190314.txt'),
    binary=False
)

# 語数, 次元数
print(len(nwjc_model.vocab), nwjc_model.vector_size)

# + id="RoOldYwgMqTH" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 89} outputId="cd9ae3b8-0a88-4547-f608-dddfcda69573"
print(nwjc_model.most_similar('平成', topn=5))

# + id="EfEdxrB3MrV_" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 89} outputId="98a162f4-698d-4341-8f9e-d84972f6caa1"
print(nwjc_model.most_similar(positive=['兄弟', '女'], negative=['男'], topn=5))

# + [markdown] id="H0HKRVgeYTZL" colab_type="text"
# # [Pythonで形態素解析器Sudachiを使う (SudachiPy)](https://ohke.hateblo.jp/entry/2019/03/09/101500)

# + [markdown] id="yRx2c7G7Y-zY" colab_type="text"
# ## [SudachiPy](https://github.com/WorksApplications/SudachiPy)

# + id="AfsWEexPYg6I" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 191} outputId="3290fbf1-b239-45fe-9782-5f1c45279708"
# !pip install SudachiPy

# + id="d7nPg4AOY5lZ" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 260} outputId="90965e5e-7bf3-44c0-fdb4-770c097016a7"
# !pip install https://object-storage.tyo2.conoha.io/v1/nc_2520839e1f9641b08211a5c85243124a/sudachi/SudachiDict_core-20200127.tar.gz

# + id="eBlKWiInZIqJ" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 295} outputId="c2aca97b-1169-48e8-ca77-978918006400"
# !sudachipy tokenize -h

# + id="psE4-uUublAZ" colab_type="code" colab={}
from sudachipy import tokenizer
from sudachipy import dictionary

tokenizer_obj = dictionary.Dictionary().create()

# + id="Jzp2caGOb0iK" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 34} outputId="00021612-37c6-4bdd-a0fc-f0cd7e2f90bd"
# Multi-granular tokenization
# using `system_core.dic` or `system_full.dic` version 20190781
# you may not be able to replicate this particular example due to dictionary you use

mode = tokenizer.Tokenizer.SplitMode.C
[m.surface() for m in tokenizer_obj.tokenize("国家公務員", mode)]
# => ['国家公務員']

# + id="hXi8W575b4aS" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 34} outputId="123097c5-80a4-4802-9104-cdf7ceb084b0"
mode = tokenizer.Tokenizer.SplitMode.B
[m.surface() for m in tokenizer_obj.tokenize("国家公務員", mode)]
# => ['国家', '公務員']

# + id="Di1jV8ltb7Tr" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 34} outputId="1bb3c22d-5e76-47e9-caad-5d393870d5db"
mode = tokenizer.Tokenizer.SplitMode.A
[m.surface() for m in tokenizer_obj.tokenize("国家公務員", mode)]
# => ['国家', '公務', '員']

# + id="1slBzS9bb-gq" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 34} outputId="c8cedf32-0c78-4e3c-8150-52e7862434ad"
# Morpheme information

m = tokenizer_obj.tokenize("食べ", mode)[0]

m.surface() # => '食べ'

# + id="rLVC2vGgcDTb" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 34} outputId="791e428b-4b30-491a-c30a-8c788cd91062"
m.dictionary_form() # => '食べる'

# + id="5rAQG-CjcFSz" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 34} outputId="df8be1b2-b4d0-42e4-a9fe-98aa4e337d99"
m.reading_form() # => 'タベ'

# + id="KjXL5LqycHA6" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 34} outputId="ed4d673c-6e9a-457f-ec0c-0baf156d607d"
m.part_of_speech() # => ['動詞', '一般', '*', '*', '下一段-バ行', '連用形-一般']

# + id="a3uzrIogcK7r" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 34} outputId="1b5d8e31-5f8e-4930-9880-c022d759a433"
# Normalization

tokenizer_obj.tokenize("附属", mode)[0].normalized_form()
# => '付属'

# + id="xgbKl8PncNE1" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 34} outputId="a301f14c-cff5-49c1-a005-ba1d0afd437c"
tokenizer_obj.tokenize("SUMMER", mode)[0].normalized_form()
# => 'サマー'

# + id="Zw8QM0YhcPeq" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 34} outputId="00c25971-9213-4990-a2df-029902c70cfc"
tokenizer_obj.tokenize("シュミレーション", mode)[0].normalized_form()
# => 'シミュレーション'

# + id="lfmIDzCTdHrS" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 243} outputId="eb7825f9-508e-4c0e-9998-86d069dcb704"
from sudachipy import tokenizer

tokenizer_obj = dictionary.Dictionary().create()
print(type(tokenizer_obj))
# <class 'sudachipy.tokenizer.Tokenizer'>

text = '友人・我孫子とスカイツリーでスパゲティを食った。'

mode = tokenizer.Tokenizer.SplitMode.C
tokens = tokenizer_obj.tokenize(text, tokenizer.Tokenizer.SplitMode.C)
print(type(tokens))
# <class 'sudachipy.morphemelist.MorphemeList'>

for t in tokens:
    print(t.surface(), t.part_of_speech(), t.reading_form(), t.normalized_form())

StarcoderdataJupyterScriptsDedupFiltered