[Python/NLP] 네이버 영화리뷰 감성분석 (2023)

신고하기

## colab 환경에서 한글 폰트 설정!sudo apt-get install -y fonts-nanum!sudo fc-cache -fv!rm ~/.cache/matplotlib -rfimport matplotlib.pyplot as pltplt.rc('font', family='NanumBarunGothic') plt.title('안녕')

## colab 환경에서 konlpy 설정 %%bashapt-get updateapt-get install g++ openjdk-8-jdk python-dev python3-devpip3 install JPype1pip3 install konlpy%env JAVA_HOME "/usr/lib/jvm/java-8-openjdk-amd64"

import pandas as pdfrom sklearn.metrics import accuracy_score, classification_report,confusion_matrix, plot_confusion_matriximport matplotlib.pyplot as pltimport seaborn as snsimport plotly.express as pximport urllib.requesturllib.request.urlretrieve("https://raw.githubusercontent.com/e9t/nsmc/master/ratings_train.txt", filename="ratings_train.txt")urllib.request.urlretrieve("https://raw.githubusercontent.com/e9t/nsmc/master/ratings_test.txt", filename="ratings_test.txt")train_df = pd.read_table('ratings_train.txt')test_df = pd.read_table('ratings_test.txt')train_df

test_df

raw_df = train_df.append(test_df, ignore_index = True)raw_df

# 결측치 확인raw_df[raw_df['document'].isna()]

# 결측치 제거raw_df = raw_df[raw_df['document'].notnull()]raw_df

# 정규화import re raw_df['document'] = raw_df['document'].apply(lambda x : re.sub(r'[^ ㄱ-ㅣ가-힣]+', " ", str(x)))raw_df

# input, output 데이터를 X, y에 각각 분할X = raw_df.drop("label", axis = 1)y = raw_df['label']X['document'], y

# 단어 길이 빈도raw_word_counts = raw_df['document'].apply(lambda x:len(x.split(' ')))

import plotly.figure_factory as fffig = px.histogram(raw_word_counts, template="plotly_dark", color=raw_word_counts, title= "문장 길이 시각화")fig.update_layout(height = 600, width = 1000, hovermode = 'closest')fig.show()

import plotly.graph_objects as gofig = go.Figure(data=[go.Histogram(y=raw_word_counts)])fig.update_layout(height = 400, width = 800, hovermode = 'closest')fig.show()

# 리뷰의 단어개수plt.figure(figsize=(15,10))plt.hist(raw_word_counts , bins=50, facecolor='r',label='train')plt.title('Log-Histogram of word count in review',fontsize=15)plt.yscale('log',nonposy='clip')plt.legend()plt.xlabel('Number of words',fontsize=15)plt.ylabel('Number of reviews', fontsize=15)

from wordcloud import WordCloud, STOPWORDSstopwords = ['영화는','아','다','그냥','진짜','너무','의','정말','영화','가','이','은','들','는','좀','잘','걍','과','도','를','으로','자','에','와','한','하다']%matplotlib inlinewordcloud1 = WordCloud(font_path='NanumBarunGothic.ttf', stopwords = stopwords, colormap='Set3', background_color = 'black', width = 800, height = 600).generate(' '.join(raw_df['document']))plt.figure(figsize = (15, 10))plt.imshow(wordcloud1)plt.axis("off")plt.show()

from sklearn.model_selection import train_test_split# train, test 9:1로 분할 비율이 가장 성능이 좋았음X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=None, shuffle=True, stratify=y)# train set 확인X_train, y_train

# test set 확인X_test, y_test

# 일단 코랩에서 메캅 설치하는 코드인데# okt가 더 쓰기 편함!sudo apt-get install curl git!bash <(curl -s https://raw.githubusercontent.com/konlpy/konlpy/master/scripts/mecab.sh)

from konlpy.tag import Mecabmecab = Mecab()def mecab_tokenizer(text): tokens = mecab.morphs(text) return tokens from konlpy.tag import Oktokt = Okt()import timestart = time.time()#---------------------------------------------from sklearn.feature_extraction.text import TfidfVectorizertfidf = TfidfVectorizer(tokenizer=mecab_tokenizer, ngram_range=(1,2), min_df=3 , max_df=0.9)tfidf.fit(X_train['document'])train_tfidf = tfidf.transform(X_train['document'])test_tfidf = tfidf.transform(X_test['document'])#----------------9:1로 나눴을때 18분 걸림-------end = time.time() print('토큰화 수행 시간 :') print(end - start)

---------------------------------------------------------------------------NameError Traceback (most recent call last)/usr/local/lib/python3.7/dist-packages/konlpy/tag/_mecab.py in __init__(self, dicpath) 76 try:---> 77 self.tagger = Tagger('-d %s' % dicpath) 78 self.tagset = utils.read_json('%s/data/tagset/mecab.json' % utils.installpath)NameError: name 'Tagger' is not definedDuring handling of the above exception, another exception occurred:Exception Traceback (most recent call last)1 frames/usr/local/lib/python3.7/dist-packages/konlpy/tag/_mecab.py in __init__(self, dicpath) 80 raise Exception('The MeCab dictionary does not exist at "%s". Is the dictionary correctly installed?\nYou can also try entering the dictionary path when initializing the Mecab class: "Mecab(\'/some/dic/path\')"' % dicpath) 81 except NameError:---> 82 raise Exception('Install MeCab in order to use it: http://konlpy.org/en/latest/install/') 83 84 def __setstate__(self, state):Exception: Install MeCab in order to use it: http://konlpy.org/en/latest/install/

from konlpy.tag import Oktokt = Okt()# okt (Open Korean text)def okt_tokenizer(text): tokens = okt.morphs(text) return tokens

import timestart = time.time()#---------------------------------------------from sklearn.feature_extraction.text import TfidfVectorizertfidf = TfidfVectorizer(tokenizer=okt_tokenizer, ngram_range=(1,2), min_df=3 , max_df=0.9)tfidf.fit(X_train['document'])train_tfidf = tfidf.transform(X_train['document'])test_tfidf = tfidf.transform(X_test['document'])#----------------9:1로 나눴을때 18분 걸림-------end = time.time() print('토큰화 수행 시간 :') print(end - start)

/usr/local/lib/python3.7/dist-packages/sklearn/feature_extraction/text.py:517: UserWarning:The parameter 'token_pattern' will not be used since 'tokenizer' is not None'토큰화 수행 시간 :1646.9149899482727

# 확인train_tfidf

<179992x135420 sparse matrix of type '<class 'numpy.float64'>'with 3303149 stored elements in Compressed Sparse Row format>

# 0,1의 비율을 동일하게 맞춤 => 업샘플링, 다운샘플링이 필요없음y_train.value_counts()

1 899960 89996Name: label, dtype: int64

y_test.value_counts()

1 100000 10000Name: label, dtype: int64

from sklearn.linear_model import LogisticRegressionfrom sklearn.metrics import accuracy_score, classification_report,confusion_matrix, plot_confusion_matrixfrom sklearn.metrics import accuracy_score, precision_score , recall_score### 로지스틱 회귀 ###start = time.time()#----------------------------------------------lr = LogisticRegression(C=5 , random_state=0)lr.fit( train_tfidf, y_train )lr_pred = lr.predict( test_tfidf )lr_acc = lr.score( test_tfidf, y_test )#----------------------------------------------print("로지스틱 회귀 training accuracy :", lr.score( train_tfidf , y_train )*100, "%")print("로지스틱 회귀 testing accuracy :", lr_acc * 100, "%")print("--------------------------------------------------------------------------")print(classification_report( y_test, lr_pred ))print("--------------------------------------------------------------------------")print(confusion_matrix( y_test, lr_pred ))print("--------------------------------------------------------------------------")#----------------------------------------------end = time.time() print('로지스틱 회귀 수행시간 :') print(end - start)plt.figure(figsize=(20, 20))plot_confusion_matrix(lr , test_tfidf , y_test , cmap='Blues')plt.title("<< Logistic Regression_박성호 >>")

로지스틱 회귀 training accuracy : 94.45419796435397 %로지스틱 회귀 testing accuracy : 86.375 %-------------------------------------------------------------------------- precision recall f1-score support 0 0.86 0.87 0.86 10000 1 0.87 0.86 0.86 10000 accuracy 0.86 20000 macro avg 0.86 0.86 0.86 20000weighted avg 0.86 0.86 0.86 20000--------------------------------------------------------------------------[[8700 1300] [1425 8575]]--------------------------------------------------------------------------로지스틱 회귀 수행시간 :14.776463508605957

# LogisticRegression Cross Validationimport numpy as npfrom sklearn.model_selection import cross_val_score, cross_validatescores_lr = cross_val_score(lr, train_tfidf, y_train , cv=5)print("로지스틱 회귀 교차 검증 정확도: {}".format(scores_lr))print("로지스틱 회귀 교차 검증 정확도:{} +/- {}".format(np.mean(scores_lr),np.std(scores_lr))) #정확도평균+-오차

로지스틱 회귀 교차 검증 정확도: [0.8587183 0.86002389 0.86143675 0.86035335 0.85776988]로지스틱 회귀 교차 검증 정확도:0.8596604325674356 +/- 0.0012829148517790696

from sklearn.model_selection import GridSearchCV #효과적인 하이퍼 파라미터 세팅을 찾아줌from sklearn.linear_model import LogisticRegressionimport numpy as nplr = LogisticRegression( random_state = 0 )params = {'C' : [3, 3.5, 4, 4.5, 5, 6], 'max_iter' : [10, 50, 100, 1000], "penalty":["l1","l2"] }lr_grid_cv = GridSearchCV(lr, param_grid=params, cv=3, scoring='accuracy', verbose=1)

GridSearchCV(cv=3, estimator=LogisticRegression(random_state=0), param_grid={'C': [3, 3.5, 4, 4.5, 5, 6], 'max_iter': [10, 50, 100, 1000], 'penalty': ['l1', 'l2']}, scoring='accuracy', verbose=1)

import timestart = time.time()#----------------------------------------------lr_grid_cv.fit(test_tfidf, y_test)print("로지스틱 최적 점수 : {}".format(lr_grid_cv.best_score_))print("로지스틱 최적 파라미터 : {}".format(lr_grid_cv.best_params_))print(lr_grid_cv.best_estimator_)#----------------------------------------------end = time.time() print("--------------------------------------------------------------------------")print('Execution time is:') print(end - start)

로지스틱 최적 점수 : 0.8149501390817049로지스틱 최적 파라미터 : {'C': 4.5, 'max_iter': 100, 'penalty': 'l2'}LogisticRegression(C=4.5, random_state=0)--------------------------------------------------------------------------Execution time is:124.36003136634827

from sklearn.naive_bayes import ComplementNBimport matplotlib.pyplot as pltstart = time.time()nb_clf = ComplementNB(alpha = 0.5)nb_clf.fit(train_tfidf, y_train)nb_pred = nb_clf.predict( test_tfidf )nb_acc = nb_clf.score( test_tfidf , y_test )print("Naive Bayes training accuracy :", nb_clf.score( train_tfidf , y_train )*100, "%")print("Naive Bayes testing accuracy :", nb_acc * 100, "%")print("--------------------------------------------------------------------------")print(classification_report(y_test, nb_pred))print("--------------------------------------------------------------------------")print(confusion_matrix(y_test, nb_pred))print("--------------------------------------------------------------------------")plot_confusion_matrix(nb_clf, test_tfidf , y_test, cmap='Greys')plt.title("Complement Naive Bayes, Acc=86.165")end = time.time() print("--------------------------------------------------------------------------")print('Execution time is:') print(end - start)

Naive Bayes training accuracy : 90.51124494421975 %Naive Bayes testing accuracy : 86.46000000000001 %-------------------------------------------------------------------------- precision recall f1-score support 0 0.86 0.88 0.87 10000 1 0.87 0.85 0.86 10000 accuracy 0.86 20000 macro avg 0.86 0.86 0.86 20000weighted avg 0.86 0.86 0.86 20000--------------------------------------------------------------------------[[8771 1229] [1479 8521]]

params = {'alpha': [0.3, 0.45, 0.5, 0.55, 0.8] }nb_clf = ComplementNB()nb_grid_cv = GridSearchCV(nb_clf, param_grid=params, cv=3, scoring='accuracy', verbose=1)import timestart = time.time()#----------------------------------------------nb_grid_cv.fit(train_tfidf, y_train)print("나이브 베이즈 최적 점수 : {}".format(nb_grid_cv.best_score_))print("나이브 베이즈 최적 파라미터 : {}".format(nb_grid_cv.best_params_))print(nb_grid_cv.best_estimator_)#----------------------------------------------end = time.time() print("--------------------------------------------------------------------------")print('Execution time is:') print(end - start)

Fitting 3 folds for each of 5 candidates, totalling 15 fits나이브 베이즈 최적 점수 : 0.8584937173974788나이브 베이즈 최적 파라미터 : {'alpha': 0.45}ComplementNB(alpha=0.45)--------------------------------------------------------------------------Execution time is:1.2268147468566895

# 0806import pandas as pdimport seaborn as snsimport matplotlib.pyplot as pltimport numpy as npmodels_acc = {'Logistic Regression':86.375 , 'Naive Bayse':86.460 , 'SVM':85.937, 'LighGBM':84.74 , 'Random Forest': 81.97 , 'K-NN':60.175 , }models_acc_df = pd.DataFrame(pd.Series(models_acc))models_acc_df.columns = ['정확도']models_acc_df['모델'] = ['Logistic Regression', 'Naive Bayse', 'SVM', 'LighGBM', 'Random Forest', 'K-NN']models_acc_df.set_index(pd.Index([1, 2 , 3 , 4 , 5 , 6]))

import plotly.express as pxfig = px.bar(models_acc_df, x='정확도', y='모델' ,color='모델', range_x=(50,90), template="plotly_dark", text_auto='.4s', title="<<심화기계학습 최종 모델평가>> by 박성호" )fig.update_traces(textfont_size=15, textangle=0, textposition="inside", cliponaxis=False)fig.update_yaxes(categoryorder="total ascending")fig.update_layout(height = 600, width = 1000, hovermode = 'closest')fig.update_layout(coloraxis = {'colorscale':'Bluered_r'})

# 새로운 텍스트를 직접 입력해 감성 예측 수행해봅시다!st = input("감성을 분석할 문장을 입력하세요: ")

1. 감성을 분석할 문장을 입력하세요: 박성호가 찢었다,,, 개꿀잼이네2. 감성을 분석할 문장을 입력하세요: 와,,, 이런 것도 상품이라고 차라리 내가 만드는 게 나을 뻔ㅋㅋ

st = re.compile(r'[ㄱ-ㅣ가-힣]+').findall(st)print(st)st = [" ".join(st)]print(st)

['박성호가', '찢었다', '개꿀잼이네']['박성호가 찢었다 개꿀잼이네']['와', '이런', '것도', '상품이라고', '차라리', '내가', '만드는', '게', '나을', '뻔ㅋㅋ']['와 이런 것도 상품이라고 차라리 내가 만드는 게 나을 뻔ㅋㅋ']

# 입력 텍스트의 벡터화st_tfidf = tfidf.transform(st)# 감성 분석 모델에 적용하여 예측st_predict = nb_clf.predict(st_tfidf)

# 예측값 출력if(st_predict ==0): print(st, "->> 부정 감성")else: print(st, "->> 긍정 감성")

1. ['박성호가 찢었다 개꿀잼이네'] ->> 긍정 감성2. ['와 이런 것도 상품이라고 차라리 내가 만드는 게 나을 뻔ㅋㅋ'] ->> 부정 감성