<jupyter_start><jupyter_text><jupyter_code>from sklearn.datasets import fetch_20newsgroups
newsdata=fetch_20newsgroups(subset='train')
print(newsdata.keys())
print( len(newsdata.data), len(newsdata.filenames), len(newsdata.target_names), len(newsdata.target))
print(newsdata.target_names)
print(newsdata.target[0])
print(newsdata.target_names[7])
print(newsdata.data[0])
# 데이터를 토큰화 -> BoW 로 만듦

from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfTransformer
from sklearn.naive_bayes import MultinomialNB # 다항분포 나이브 베이즈 모델
from sklearn.metrics import accuracy_score # 정확도 계산
dtmvector = CountVectorizer()
X_train_dtm = dtmvector.fit_transform(newsdata.data)
print(X_train_dtm.shape)
tdidf_transformer = TfidfTransformer()
tfidfv = tdidf_transformer.fit_transform(X_train_dtm)
print(tfidfv.shape)
mod = MultinomialNB()
mod.fit(tfidfv, newsdata.target) # X_train , Y_train
newsdata_test = fetch_20newsgroups(subset='test',shuffle=True)
X_test_dtm = dtmvector.transform(newsdata_test.data) # DTM으로 변환
tfidfv_test = tdidf_transformer.transform(X_test_dtm) #Tf-idf 행렬로 변환
predicted = mod.predict(tfidfv_test) #예측
print("정확도:", accuracy_score(newsdata_test.target, predicted)) #예측값과 실제값 비교<jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup