# -*- coding: utf-8 -*-
"""online_supplements.ipynb

### Machine Learning and Explainable Artificial Intelligence application for Multi-Fabrication Data Analysis in Semiconductor Manufacturing ###
### Youjin L, Yonghan R ###
### supplementary Materials ###

# import libraries
import pandas as pd
import numpy as np
import codecs
import matplotlib.pyplot as plt
import seaborn as sns
#preprocessing
from sklearn.preprocessing import MinMaxScaler, StandardScaler, OneHotEncoder
from sklearn.impute import SimpleImputer
from sklearn.model_selection import train_test_split
from sklearn.model_selection import ShuffleSplit
from sklearn.model_selection import cross_validate
from sklearn.metrics import mean_squared_error, mean_absolute_error
#regressor, XAI
from sklearn.model_selection import RandomizedSearchCV
from sklearn.linear_model import Lasso
from sklearn.ensemble import AdaBoostRegressor
from sklearn.ensemble import RandomForestRegressor
from sklearn.neighbors import KNeighborsRegressor
from sklearn.svm import SVR
from sklearn.gaussian_process.kernels import (RBF, Matern, RationalQuadratic,ExpSineSquared, DotProduct,ConstantKernel)
from sklearn.gaussian_process import GaussianProcessRegressor
from sklearn.linear_model import RANSACRegressor
from sklearn.neural_network import MLPRegressor
import xgboost
from lightgbm import LGBMRegressor
import shap
shap.initjs()

# the fabrication data are exclusively applicable to the study.
data_index = "LOT_WF"
target_column = "YLD"
path = "the path"
doc = codecs.open(path,'rU','UTF-16')
df = pd.read_csv(doc, sep='\t')
df.drop_duplicates(subset = data_index, keep = False, inplace = True)

#dimension reduction
def correlation(dataset, threshold):
  col_corr = set() # Set of all the names of deleted columns
  corr_matrix = dataset.corr()
  for i in range(len(corr_matrix.columns)):
    for j in range(i):
      if (corr_matrix.iloc[i, j] >= threshold) and (corr_matrix.columns[j] not in col_corr):
        colname = corr_matrix.columns[i] # getting the name of column
        col_corr.add(colname)
      if colname in dataset.columns:
        del dataset[colname] # deleting the column from the dataset
  return dataset

#data preprocessing
df_y = df[[data_index,target_column]].set_index([data_index], drop = True, inplace = False)
df = df.set_index([data_index], drop = True, inplace = False)
num_attribs = [col for col in df.columns if ('_T' in col) or ('_P' in col)] # time span and sensor parameter of processes
cat_attribs = [col for col in df.columns if ('_R_' in col) or ('_U_' in col)] # process operating conditions and equipment units of processes
df_num = df[num_attribs]
df_cat = df[cat_attribs]
mM = MinMaxScaler()
std = StandardScaler()
oh = OneHotEncoder()
pre_num = mM.fit_transform(df_num)
pre_cat = oh.fit_transform(df_cat)
pre_y = std.fit_transform(df_y)

cat_feat = []
for step, i in zip(df_cat.columns, oh.categories_):
  for a in i:
    c = str(step)+str(a)
    cat_feat.append(c)

pre_num = StandardScaler().fit_transform(pre_num)
si = SimpleImputer(fill_value=0)
pre_si_num = si.fit_transform(pre_num)
pre_cat = StandardScaler().fit_transform(pre_cat.toarray())
pre_df_num = pd.DataFrame(pre_si_num, columns = df_num.columns)
pre_df_cat = pd.DataFrame(pre_cat, columns = cat_feat)
pd_X = pd.concat([pre_df_num, pre_df_cat], axis =1)
pd_y = pd.Series(pre_y.reshape(-1))
#pearson 0.95 simple split
pc_X = correlation(pd_X, 0.95)
X_train, X_test, y_train, y_test = train_test_split(pc_X, pd_y, test_size = 0.2, random_state = 0)

#train test target distribution confirmation
ax1 = sns.distplot(y_test, label = "y_test", hist = False)
ax2 = sns.distplot(y_train, label = "y_train", ax = ax1, hist = False)
plt.show()

# tuning hyper-parameter
model_cvs = {}
model_scores ={}
n_iter =24
cv = ShuffleSplit(n_splits=20, test_size=0.25, random_state=7)
def reg_rmse(name, reg, X_train, y_train):
  scores = cross_validate(reg, X_train, y_train, cv=cv,scoring=('neg_mean_squared_error', 'neg_mean_absolute_error'), return_train_score=True)
  model_cvs[name] = reg
  model_scores[name] = scores

params = {'n_estimators': [400, 500], 'max_depth': [7, 9, 10, 11], 'min_samples_leaf': [2,3], 'max_features' :["sqrt", 1.0],'max_samples' :[0.8, 0.9], 'random_state' :[42]}
rf = RandomForestRegressor()
rfrs = RandomizedSearchCV(rf, params, random_state=7, cv=cv, n_iter=n_iter, scoring='neg_mean_squared_error')
rfrs.fit(X_train, y_train)
print(rfrs.best_params_)
best_RF = rfrs.best_estimator_
reg_rmse("RandomForest", best_RF, X_train, y_train)

adar = AdaBoostRegressor()
parameters ={'n_estimators' :[50, 100, 300, 700], 'learning_rate' : [1, 0.1, 0.01], 'loss' :['linear', 'square'] }
adar_grid = RandomizedSearchCV(adar, parameters, random_state=7, cv=cv, n_iter=n_iter, scoring='neg_mean_squared_error')
adar_grid.fit(X_train, y_train)
print(adar_grid.best_params_)
best_adar = adar_grid.best_estimator_
reg_rmse("Ada Boost", best_adar, X_train, y_train)

params = {'n_estimators': [200, 400,700],'learning_rate': [0.01], 'max_depth': [9,5,3], 'colsample_bytree': [0.6, 0.5, 0.4], 'subsample': [0.2, 0.3],}
#cv = ShuffleSplit(n_splits=5, test_size=0.2, random_state=7)
lgbm = LGBMRegressor()
LGBMRrs = RandomizedSearchCV(lgbm, params, random_state=7, cv=cv, n_iter=n_iter, scoring='neg_mean_squared_error')
LGBMRrs.fit(X_train, y_train)
print(LGBMRrs.best_params_)
best_lgbm = LGBMRrs.best_estimator_
reg_rmse("LGBM", best_lgbm, X_train, y_train)

xgb = xgboost.XGBRegressor()
parameters = {'objective' :['reg:squarederror'], 'n_estimators' : [100, 300, 700], 'learning_rate' : [0.01, 0.03], 'gamma' :[0, 0.5], 'subsample' : [0.25, 0.75], 'max_depth' :[6, 8, 10]}
xgb_grid = RandomizedSearchCV(xgb, parameters, random_state=7, cv=cv, n_iter=n_iter, scoring='neg_mean_squared_error')
xgb_grid.fit(X_train, y_train)
print(xgb_grid.best_params_)
best_xgb = xgb_grid.best_estimator_
reg_rmse("XGB", best_xgb, X_train, y_train)

params = {'alpha': [0.1, 0.5], 'max_iter': [1000, 2000], 'tol': [0.0001,0.0005,0.001], 'selection': ['cyclic', 'random']}
lasso = Lasso()
lassors = RandomizedSearchCV(lasso, params, random_state=7, cv=cv, n_iter=n_iter, scoring='neg_mean_squared_error')
lassors.fit(X_train, y_train)
print(lassors.best_params_)
best_lasso = lassors.best_estimator_
reg_rmse("Lasso", best_lasso, X_train, y_train)

neigh = KNeighborsRegressor()
parameters ={'n_neighbors' :[ 6, 7, 8], 'weights' : ['uniform', 'distance'], 'p' :[1, 2], 'algorithm' : ['auto', 'kd_tree']  }
n_iter =24
cv = ShuffleSplit(n_splits=20, test_size=0.25, random_state=7)
neigh_grid = RandomizedSearchCV(neigh, parameters, random_state=7, cv=cv, n_iter=n_iter, scoring='neg_mean_squared_error')
neigh_grid.fit(X_train, y_train)
print(neigh_grid.best_params_)
best_neigh = neigh_grid.best_estimator_
reg_rmse("Kneighbors_CV", best_neigh, X_train, y_train)

params = {'kernel': [ 'rbf', 'linear', 'sigmoid'], 'gamma' :['scale', 'auto'], 'coef0' : [0, 0.5],  'C' :[1, 0.5]}
svr = SVR()
svr_rs = RandomizedSearchCV(svr, params, random_state=7, cv=cv, n_iter=n_iter, scoring='neg_mean_squared_error')
svr_rs.fit(X_train, y_train)
print(svr_rs.best_params_)
best_svr = svr_rs.best_estimator_
reg_rmse("SVR", best_svr, X_train, y_train)

params = {'min_samples': [ 2, 3, 5, 7], 'stop_probability' :[0.99, 0.999], 'max_trials' :[300, 500, 1000]}
ransacr = RANSACRegressor()
ransacr_rs = GridSearchCV(ransacr, params,  cv=cv,  scoring='neg_mean_squared_error')
ransacr_rs.fit(X_train, y_train)
print(ransacr_rs.best_params_)
best_ransacr = ransacr_rs.best_estimator_
reg_rmse("RANSAC", best_ransacr, X_train, y_train)

kernel = kernels.ConstantKernel(2.0, (1e-1, 1e3)) * kernels.RBF(2.0, (1e-3, 1e3))
kernel2=RationalQuadratic()
kernel3 = kernels.ConstantKernel(1.0, (1e-1, 1e3)) * kernels.RBF(1.0, (1e-3, 1e3))
params = { 'alpha' :[0.1, 0.01, 0.5], 'kernel' : [kernel, kernel2, kernel3]}
gpr = GaussianProcessRegressor()
gpr_rs = RandomizedSearchCV(gpr, params, random_state=7, cv=cv, n_iter=n_iter, scoring='neg_mean_squared_error')
gpr_rs.fit(X_train, y_train)
print(gpr_rs.best_params_)
best_gpr = gpr_rs.best_estimator_
pred = best_gpr.predict(X_test)
reg_rmse("GPR", best_gpr, X_train, y_train)

parameters ={'hidden_layer_sizes' :[(50,), (100,), (200,), (200,2)], 'activation' : ['relu', 'logistic'], 'max_iter' :[200 , 300, 100] }
regr = MLPRegressor()
regr_grid = RandomizedSearchCV(regr, parameters, random_state=7, cv=cv, n_iter=n_iter, scoring='neg_mean_squared_error')
regr_grid.fit(X_train, y_train)
print(regr_grid.best_params_)
best_regr = regr_grid.best_estimator_
reg_rmse("MLP", best_regr, X_train, y_train)

names = ['Ada Boost', 'LGBM', 'XGB', 'RandomForest', 'Lasso', "Kneighbors_CV", "SVR", "RANSAC", "GPR",]
for name in names:
  print(pd.DataFrame(model_scores[name]))

#Prediction
names = ['Ada Boost', 'LGBM', 'XGB', 'RandomForest', 'Lasso', "Kneighbors_CV", "SVR", "RANSAC", "GPR", "MLP"]
predictions ={}
rmses = {}
maes = {}
final_model = {}
for name in names:
  rg = model_cvs[name]
  rg.fit(X_train, y_train)
  final_model[name] = rg
  pred = rg.predict(X_test)
  predictions[name] = pred

mse = mean_squared_error(pred, y_test)
mae = mean_absolute_error(pred, y_test)
rmses[name] = np.sqrt(mse)
maes[name] = mae


# SHAP analysis with visualization
rf = final_model['RandomForest']
explainer = shap.TreeExplainer(rf)
shap_values = explainer.shap_values(X_train)
shap.summary_plot(shap_values, X_train)

explainer1 = shap.Explainer(rf, X_train)
shap_values1 = explainer1(X_train, check_additivity = False)

shap.plots.waterfall(shap_values1[0])
shap.plots.waterfall(shap_values1[17])
shap.plots.waterfall(shap_values1[20])
shap.plots.scatter(shap_values1[:,"step_aa_T"])
shap.plots.scatter(shap_values1[:,"step_s_R_53"])
shap.plots.scatter(shap_values1[:,"step_s_T"])
shap.plots.scatter(shap_values1[:,"step_ax_P"])
shap.plots.scatter(shap_values1[:,"step_ac_U_8"])
shap.plots.scatter(shap_values1[:,"step_ac_P"])