<jupyter_start><jupyter_text>Author: [<NAME>](https://www.linkedin.com/in/yuntingchiu/)<jupyter_code>import cv2
import matplotlib.pyplot as plt
import numpy as np
import time
import pandas as pd
#wd
%cd /content/drive/MyDrive/American_University/2021_Fall/DATA-793-001_Data Science Practicum/data
!pwd<jupyter_output>/content/drive/MyDrive/American_University/2021_Fall/DATA-793-001_Data Science Practicum/data
/content/drive/MyDrive/American_University/2021_Fall/DATA-793-001_Data Science Practicum/data<jupyter_text>Exploratory Data AnalysisRead the data (`.npz` file)<jupyter_code>"""
data_zipped = np.load("np_data_all.npz", allow_pickle=True)

for item in data_zipped.files:
    print(item)
    print(data_zipped[item])
    
print(data_zipped[item].shape)
data = data_zipped[item]
"""<jupyter_output>arr_0
[[array([134, 131, 116, ...,  68,  60,  71], dtype=uint8) 'fake']
 [array([133, 130, 115, ...,  71,  59,  71], dtype=uint8) 'fake']
 [array([117, 113, 112, ...,  43,  31,  45], dtype=uint8) 'fake']
 ...
 [array([ 33,  20,  66, ..., 188, 155, 172], dtype=uint8) 'real']
 [array([ 51,  28,  46, ..., 116,  52,  50], dtype=uint8) 'real']
 [array([174, 140, 102, ...,  23,  39,  98], dtype=uint8) 'real']]
(13984, 2)<jupyter_text>Read the data (`.npy` file)<jupyter_code>data = np.load("np_data_one.npy", allow_pickle=True)<jupyter_output><empty_output><jupyter_text>Check the length of $X$ and $y$<jupyter_code>X = []
y = []
for i in data:
  X.append(i[0])
  y.append(i[1])
print(len(X))
print(len(y))
print("The length should be " + str((6984+7000)))
print(X)
print(y)
print("data dimension:",data.shape)<jupyter_output>[array([133, 130, 115, ...,  71,  59,  71], dtype=uint8), array([ 49,  40,  25, ..., 111, 105,  73], dtype=uint8), array([28, 21, 15, ..., 49, 44, 40], dtype=uint8), array([ 61,  61,  53, ..., 244, 246, 243], dtype=uint8), array([15, 14, 19, ..., 88, 66, 53], dtype=uint8), array([ 1,  1,  1, ..., 18, 14, 11], dtype=uint8), array([91, 61, 63, ..., 29, 20, 67], dtype=uint8), array([250, 187, 143, ..., 100,  99, 104], dtype=uint8), array([26, 25, 31, ..., 14, 15, 20], dtype=uint8), array([36, 32, 20, ..., 38, 30, 17], dtype=uint8), array([ 51,  37,  34, ..., 179, 158, 131], dtype=uint8), array([163, 146, 164, ..., 109,  55, 104], dtype=uint8), array([ 63,  58,  65, ..., 255, 247, 253], dtype=uint8), array([ 53,  41,  29, ..., 164, 180, 196], dtype=uint8), array([110,  32,  32, ..., 188, 149, 180], dtype=uint8), array([29, 18, 52, ..., 13, 14,  8], dtype=uint8), array([ 78,  59,  65, ..., 194, 207, 239], dtype=uint8), array([101,  77,  73, ...,  34,  32,  46], dtype=uint8), array([ 83,  77[...]<jupyter_text>Visualization<jupyter_code>fake_cnt = 0
real_cnt = 0
for i in data:
  if i[1] == "fake":
    fake_cnt += 1
  else:
    real_cnt += 1

#print(fake_cnt)
#print(real_cnt)
df = [['fake', fake_cnt], ['real', real_cnt]]
df = pd.DataFrame(df, columns=['image_type', 'count'])
#ax = df.plot.bar(x='video_type', y='count', rot=0)
#fig = plt.figure()
plt.bar(df['image_type'], df['count'])
plt.xlabel("Image Type")
plt.ylabel("Count")
plt.savefig('count_type.png')<jupyter_output><empty_output><jupyter_text>Machine Learning Task<jupyter_code>from sklearn.model_selection import train_test_split
from sklearn.svm import SVC
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import StandardScaler # standardize features by removing the mean and scaling to unit variance.
from sklearn.metrics import confusion_matrix
#from sklearn.metrics import plot_confusion_matrix
from sklearn.metrics import accuracy_score
from sklearn.metrics import classification_report
from sklearn.ensemble import RandomForestClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import roc_curve
from sklearn.metrics import roc_auc_score<jupyter_output><empty_output><jupyter_text>Support Vector Machine<jupyter_code>start_time = time.time()
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.2, random_state = 42) # 80% for training, 20 for of testing
svm_clf = make_pipeline(StandardScaler(), SVC(gamma='scale', C = 1)) # clf = classifer
svm_clf.fit(X_train, y_train)
y_pred = svm_clf.predict(X_test)

print("--- %s seconds ---" % (time.time() - start_time))
print(confusion_matrix(y_test, y_pred))<jupyter_output><empty_output><jupyter_text>SVM Confusion Matrix<jupyter_code>#plot_confusion_matrix(svm_clf, X_test, y_test, values_format = '.0f') 
#plt.figure(figsize=(12,8))
#plt.show()
conf_matrix = confusion_matrix(y_true = y_test, y_pred = y_pred)
# Print the confusion matrix using Matplotlib

fig, ax = plt.subplots(figsize=(7.5, 7.5))
ax.matshow(conf_matrix, cmap=plt.cm.Blues, alpha=0.3)
for i in range(conf_matrix.shape[0]):
    for j in range(conf_matrix.shape[1]):
        ax.text(x=j, y=i,s=conf_matrix[i, j], va='center', ha='center', size='xx-large')
 
plt.xlabel('Predictions', fontsize=18)
plt.ylabel('Actuals', fontsize=18)
plt.title('Confusion Matrix', fontsize=18)
plt.show()
plt.savefig('Confusion_Matrix.png')<jupyter_output><empty_output><jupyter_text>ROC curves- ROC Curves summarize the trade-off between the true positive rate and false positive rate for a predictive model using different probability thresholds.- Precision-Recall curves summarize the trade-off between the true positive rate and the positive predictive value for a predictive model using different probability thresholds.- ROC curves are appropriate when the observations are balanced between each class, whereas precision-recall curves are appropriate for imbalanced datasets.<jupyter_code>"""
# generate a no skill prediction (majority class)
ns_probs = [0 for _ in range(len(y_test))]
lr_probs = svm_clf.predict_proba(X_test)
# keep probabilities for the positive outcome only
lr_probs = lr_probs[:, 1]
# calculate scores
ns_auc = roc_auc_score(y_test, ns_probs)
lr_auc = roc_auc_score(y_test, lr_probs)
# summarize scores
print('No Skill: ROC AUC=%.3f' % (ns_auc))
print('Logistic: ROC AUC=%.3f' % (lr_auc))
# calculate roc curves
ns_fpr, ns_tpr, _ = roc_curve(y_test, ns_probs)
lr_fpr, lr_tpr, _ = roc_curve(y_test, lr_probs)
# plot the roc curve for the model
plt.plot(ns_fpr, ns_tpr, linestyle='--', label='No Skill')
plt.plot(lr_fpr, lr_tpr, marker='.', label='Logistic')
# axis labels
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
# show the legend
plt.legend()
# show the plot
plt.show()
plt.savefig('ROC_AUC_Plot.png')
"""<jupyter_output><empty_output><jupyter_text>SVM Accuracy Score<jupyter_code>print("----------Accuracy Score----------------")
print(accuracy_score(y_test, y_pred))

target_names = ['fake', 'real']
print(classification_report(y_test, y_pred, target_names=target_names))<jupyter_output>----------Accuracy Score----------------
0.6475
              precision    recall  f1-score   support

        fake       0.68      0.60      0.64       207
        real       0.62      0.70      0.66       193

    accuracy                           0.65       400
   macro avg       0.65      0.65      0.65       400
weighted avg       0.65      0.65      0.65       400<jupyter_text>Random Forest Classifier<jupyter_code>start_time = time.time()
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.2, random_state = 42) # 80% for training, 20 for of testing
#rf_clf = RandomForestClassifier(n_estimators=100, random_state=42, bootstrap=True)
#rf_clf.fit(X_train, y_train)
#y_pred = rf_clf.predict(X_test)

#print("--- %s seconds ---" % (time.time() - start_time))
#print(confusion_matrix(y_test, y_pred))<jupyter_output><empty_output><jupyter_text>Random Forest Accuracy Score<jupyter_code>print(accuracy_score(y_test, y_pred))<jupyter_output>0.7257142857142858<jupyter_text>Logistic Regression<jupyter_code>start_time = time.time()
lg_clf = LogisticRegression(random_state=42, C=1)
lg_clf.fit(X_train, y_train)
y_pred = lg_clf.predict(X_test)

print("--- %s seconds ---" % (time.time() - start_time))
print(confusion_matrix(y_test, y_pred))<jupyter_output>/usr/local/lib/python3.7/dist-packages/sklearn/linear_model/_logistic.py:940: ConvergenceWarning: lbfgs failed to converge (status=1):
STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.

Increase the number of iterations (max_iter) or scale the data as shown in:
    https://scikit-learn.org/stable/modules/preprocessing.html
Please also refer to the documentation for alternative solver options:
    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression
  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)<jupyter_text>Logistic Regression Accuracy Score<jupyter_code>print(accuracy_score(y_test, y_pred))<jupyter_output>0.6528571428571428<jupyter_text>Nested Cross-Validation (Testing Zone)<jupyter_code>from sklearn.datasets import make_classification
from sklearn.model_selection import KFold
from sklearn.model_selection import GridSearchCV
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import accuracy_score
# manual nested cross-validation for random forest on a classification dataset
from numpy import mean
from numpy import std
from sklearn.datasets import make_classification
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import KFold
from sklearn.model_selection import GridSearchCV
from sklearn.ensemble import RandomForestClassifier
# create dataset

#X, y = make_classification(n_samples=1000, n_features=20, random_state=1, n_informative=10, n_redundant=10)
#print(X.shape)
#print(y.shape)

# configure the cross-validation procedure
cv_inner = KFold(n_splits=3, shuffle=True, random_state=1)
# define the model
model = RandomForestClassifier(random_state=42)
# define search space
space = dict()
space['n_estimators'] = [10, 100, 500]
#space['max_features'] = [2, 4, 6]
# define search
search = GridSearchCV(model, space, scoring='accuracy', n_jobs=1, cv=cv_inner, refit=True)
# configure the cross-validation procedure
cv_outer = KFold(n_splits=10, shuffle=True, random_state=1)
# execute the nested cross-validation
scores = cross_val_score(search, X_train, y_train, scoring='accuracy', cv=cv_outer, n_jobs=-1)
# report performance
print('Accuracy: %.3f (%.3f)' % (mean(scores), std(scores)))


result = search.fit(X_train, y_train)
# get the best performing model fit on the whole training set
best_model = result.best_estimator_
# evaluate model on the hold out dataset
yhat = best_model.predict(X_test)
space = {}
space['n_estimators'] = list(range(1, 1001))
print(space)<jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup