<jupyter_start><jupyter_text>$$entropy=-\sum_{i=1}^NP_i\cdot\log_2P_i$$<jupyter_code>def entropy(arr):
    length = sum(arr)
    probs =  [i / length for i in arr]
    entropy = sum([-i * log(i) for i in probs])
    return entropy

entropy([8, 0.1e-100])
x = np.linspace(0.1e-10, 1 - 0.1e-10, 100)
data =  - x * np.log(x) - (1 - x) * np.log(1 - x)
plt.plot(x, data, "g-")
mask = np.argmax(data)
plt.plot(x[mask], data[mask], "*r")
plt.grid()
plt.ylabel("entropy")
plt.xlabel("proba")
plt.title("entropy in binary case")
plt.tight_layout()<jupyter_output><empty_output><jupyter_text>$$gini = 1 - \sum_{i=1}^N\cdot P_i^2$$<jupyter_code>def gini(arr):
    length = sum(arr)
    probs =  [i / length for i in arr]
    gini = 1 - sum([i ** 2 for i in probs])
    return gini

gini([19, 0, 0])
x = np.linspace(0.1e-10, 1 - 0.1e-10, 100)
data = 1 - ((1 - x) ** 2 + x ** 2)
mask = np.argmax(data)
plt.plot(x, data, "g-")
plt.plot(x[mask], data[mask], "*r")
plt.grid()
plt.ylabel("gini")
plt.xlabel("proba")
plt.title("gini in binary case")
plt.tight_layout()
data = [
    {"level":"Senior", "lang":"Java",   "Instagram":"Yes", "status":"True" },
    {"level":"Senior", "lang":"Java",   "Instagram":"No",  "status":"False"},
    {"level":"Mid",    "lang":"Python", "Instagram":"No",  "status":"True" },
    {"level":"Junior", "lang":"Python", "Instagram":"No",  "status":"False"},
    {"level":"Junior", "lang":"R",      "Instagram":"No",  "status":"False"},
    {"level":"Junior", "lang":"R",      "Instagram":"Yes", "status":"True" },
    {"level":"Mid",    "lang":"R",      "Instagram":"Yes", "status":"True" },
    {"level":"Senior", "lang":"Python", "Instagram":"No",  "status":"False"},
    {"level":"Senior", "lang":"R",      "Instagram":"No",  "status":"False"},
    {"level":"Junior", "lang":"Python", "Instagram":"Yes", "status":"False"},
    {"level":"Senior", "lang":"Python", "Instagram":"Yes", "status":"True" },
    {"level":"Mid",    "lang":"Python", "Instagram":"No",  "status":"False"},
]
df = pd.DataFrame(data)
df
# split by Instagram
df.sort_values("Instagram")
yes_true = len(df[(df["Instagram"] == "Yes") & (df["status"] == "True")])
yes_false = len(df[(df["Instagram"] == "Yes") & (df["status"] == "False")])

no_true = len(df[(df["Instagram"] == "No") & (df["status"] == "True")])
no_false = len(df[(df["Instagram"] == "No") & (df["status"] == "False")])

print(f"yes_true = {yes_true}, yes_false = {yes_false}, no_true = {no_true}, no_false = {no_false}")
print(f"entropy/yes = {entropy([yes_true, yes_false])}, entropy/no = {entropy([no_true, no_false])}")
print(f"gini/yes = {gini([yes_true, yes_false])}, gini/no = {gini([no_true, no_false])}")
total_entropy = entropy([yes_true, yes_false]) * (yes_true + yes_false) / (yes_true + yes_false + no_true + no_false)
total_entropy += entropy([no_true, no_false]) * (no_true + no_false) / (yes_true + yes_false + no_true + no_false)
print(f"total entropy = {total_entropy}")
# split by lang
df.sort_values("lang")
python_true = len(df[(df["lang"] == "Python") & (df["status"] == "True")])
python_false = len(df[(df["lang"] == "Python") & (df["status"] == "False")])

r_true = len(df[(df["lang"] == "R") & (df["status"] == "True")])
r_false = len(df[(df["lang"] == "R") & (df["status"] == "False")])

java_true = len(df[(df["lang"] == "Java") & (df["status"] == "True")])
java_false = len(df[(df["lang"] == "Java") & (df["status"] == "False")])

print(f"python_true = {python_true}, python_false = {python_false}, r_true = {r_true}, r_false = {r_false}, java_true = {java_true}, java_false = {java_false}")
print(f"entropy/python = {entropy([python_true, python_false])}, entropy/r = {entropy([r_true, r_false])}, entropy/java = {entropy([java_true, java_false])}")
print(f"gini/python = {gini([python_true, python_false])}, gini/r = {gini([r_true, r_false])}, gini/java = {gini([java_true, java_false])}")
total_entropy = entropy([python_true, python_false]) * (python_true + python_false) / (python_true + python_false + r_true + r_false + java_true + java_false)
total_entropy += entropy([r_true, r_false]) * (r_true + r_false) / (python_true + python_false + r_true + r_false + java_true + java_false)
total_entropy += entropy([java_true, java_false]) * (java_true + java_false) / (python_true + python_false + r_true + r_false + java_true + java_false)
print(f"total entropy = {total_entropy}")
# split by level
df.sort_values("level")
junior_true = len(df[(df["level"] == "Junior") & (df["status"] == "True")])
junior_false = len(df[(df["level"] == "Junior") & (df["status"] == "False")])

mid_true = len(df[(df["level"] == "Mid") & (df["status"] == "True")])
mid_false = len(df[(df["level"] == "Mid") & (df["status"] == "False")])

senior_true = len(df[(df["level"] == "Senior") & (df["status"] == "True")])
senior_false = len(df[(df["level"] == "Senior") & (df["status"] == "False")])

print(f"junior_true = {junior_true}, junior_false = {junior_false}, mid_true = {mid_true}, mid_false = {mid_false}, senior_true = {senior_true}, senior_false = {senior_false}")
print(f"entropy/junior = {entropy([junior_true, junior_false])}, entropy/mid = {entropy([mid_true, mid_false])}, entropy/senior = {entropy([senior_true, senior_false])}")
print(f"gini/junior = {gini([junior_true, junior_false])}, gini/mid = {gini([mid_true, mid_false])}, gini/senior = {gini([senior_true, senior_false])}")
total_entropy = entropy([junior_true, junior_false]) * (junior_true + junior_false) / (junior_true + junior_false + mid_true + mid_false + senior_true + senior_false)
total_entropy += entropy([mid_true, mid_false]) * (mid_true + mid_false) / (junior_true + junior_false + mid_true + mid_false + senior_true + senior_false)
total_entropy += entropy([senior_true, senior_false]) * (senior_true + senior_false) / (junior_true + junior_false + mid_true + mid_false + senior_true + senior_false)
print(f"total entropy = {total_entropy}")
import pydotplus
from sklearn.tree import DecisionTreeClassifier, export_graphviz
from sklearn.datasets import load_iris
from IPython.display import Image

iris = load_iris()
data = iris.data
target = iris.target

model = DecisionTreeClassifier(criterion="entropy")
model.fit(data, target);
dot_data = export_graphviz(model, feature_names=iris.feature_names, class_names=iris.target_names)
graph = pydotplus.graph_from_dot_data(dot_data)
Image(graph.create_png())
# https://graphviz.gitlab.io/_pages/Download/Download_windows.html
# then add this in the path enviroment variable
df['Instagram'] = df['Instagram'].replace(["Yes", "No"], [1, 0])
df["lang"] = df["lang"].replace(["Java", "Python", "R"], [1, 2, 3])
df["level"] = df["level"].replace(["Senior", "Mid", "Junior"], [3, 2, 1]) # HW !!
df["status"] = df["status"].replace(["True", "False"], [1, 0])
df
model = DecisionTreeClassifier()
model.fit(df.iloc[:, :3], df.iloc[:,3]);
dot_data = export_graphviz(model, feature_names=df.columns[:-1], class_names=["reject", "accept"])
graph = pydotplus.graph_from_dot_data(dot_data)
Image(graph.create_png())
plt.bar(range(4), model.feature_importances_, tick_label=iris.feature_names)
plt.grid()
plt.tight_layout()<jupyter_output><empty_output><jupyter_text>---<jupyter_code>from sklearn.tree import DecisionTreeClassifier
from sklearn.datasets import load_digits
from sklearn.model_selection import train_test_split

digit = load_digits()
data = digit.data
target = digit.target

x_train, x_test, y_train, y_test = train_test_split(data, target)

model = DecisionTreeClassifier(max_depth=7).fit(x_train, y_train)
model.score(x_test, y_test)
from sklearn.ensemble import RandomForestClassifier

model = RandomForestClassifier(n_estimators=5).fit(load_iris().data, load_iris().target)
# model.score(x_test, y_test)
plt.bar(range(4), model.feature_importances_, tick_label=iris.feature_names)
plt.grid()
plt.tight_layout()<jupyter_output><empty_output><jupyter_text>---<jupyter_code>from sklearn.preprocessing import PolynomialFeatures
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import train_test_split
from sklearn.datasets import load_digits

data = load_digits()
features = data.data
target = data.target

x_train, x_test, y_train, y_test = train_test_split(features, target)
poly_x_train = PolynomialFeatures().fit_transform(x_train)

model1 = RandomForestClassifier().fit(x_train, y_train)
model2 = RandomForestClassifier().fit(poly_x_train, y_train)

print(model1.score(x_test, y_test))
poly_x_test = PolynomialFeatures().fit_transform(x_test)
print(model2.score(poly_x_test, y_test))
from sklearn.model_selection import cross_val_score
cross_val_score(RandomForestClassifier(n_estimators=40, max_depth=31), features, target, cv=5).mean()
x_train.shape
poly_x_train.shape<jupyter_output><empty_output><jupyter_text>---<jupyter_code>from sklearn.preprocessing import StandardScaler, MinMaxScaler
data = np.random.randint(0, 5, 10).reshape((-1,1))
ss = StandardScaler().fit(data)
print(ss.mean_)
print(ss.scale_)
ss.transform(data)
data
from sklearn.model_selection import GridSearchCV
grid = GridSearchCV(estimator=RandomForestClassifier(), verbose=1,
             param_grid={"n_estimators":range(5, 50, 5), "max_depth":range(5, 50, 2)})
data = PolynomialFeatures().fit_transform(features)
grid.fit(data, target)
grid.best_params_
grid.best_score_
1000
200 200 200 200 200<jupyter_output><empty_output><jupyter_text>---<jupyter_code>from sklearn.datasets import make_classification
from sklearn.ensemble import RandomForestClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.model_selection import train_test_split
from sklearn.model_selection import GridSearchCV
from sklearn.preprocessing import PolynomialFeatures

data, label = make_classification(n_samples=300, n_features=10, n_informative=2, n_classes=2)
model = RandomForestClassifier()
param = {
    "n_estimators":range(5, 50, 5),
    "max_depth":range(5, 50, 2)
}
grid = GridSearchCV(estimator=model, param_grid=param)
grid.fit(data, label);
data4bar = grid.best_estimator_.feature_importances_ 
plt.bar(range(len(data4bar)), data4bar)
plt.grid()
plt.tight_layout()
grid.best_params_
grid.best_score_<jupyter_output><empty_output><jupyter_text>---<jupyter_code>extended_data  = PolynomialFeatures().fit()<jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup