# Remove the index X1, and F_lu3, F_lu6 and F_lu10. Remove rows with lu2006=2, lu2006=6 and lc2010=6
training_SA_To_5_full = training_SA_To_5_full[-which(training_SA_To_5_full$lu2006==2 | training_SA_To_5_full$lu2006 ==6 | training_SA_To_5_full$lc2010 ==6),
                                              -c(1,25,28,32)]
testing_SA_To_5_full = testing_SA_To_5_full[,-c(1,25,28,32)]

# make LU and LC factor
training_SA_To_5_full$lc2006 = factor(training_SA_To_5_full$lc2006)
training_SA_To_5_full$lu2006 = factor(training_SA_To_5_full$lu2006)
training_SA_To_5_full$lc2010 = factor(training_SA_To_5_full$lc2010)

testing_SA_To_5_full$lc2006 = factor(testing_SA_To_5_full$lc2006)
testing_SA_To_5_full$lu2006 = factor(testing_SA_To_5_full$lu2006)
testing_SA_To_5_full$lc2010 = factor(testing_SA_To_5_full$lc2010)

dim(training_SA_To_5_full) # 367  31
dim(testing_SA_To_5_full) # 159  31

########################################################################################
# 10-fold Cross-validation
########################################################################################
set.seed(123)
SA_To_5_folds <- createFolds(training_SA_To_5_full$status, k = 10, list = TRUE, returnTrain = TRUE)

tic("model fitting")
# 1st fold
set.seed(123)
SA_To_5_obj1 = Surv(training_SA_To_5_full[SA_To_5_folds[[1]],]$time, 
                    training_SA_To_5_full[SA_To_5_folds[[1]],]$status)
Cox_SA_To_5_fld1 = coxph(SA_To_5_obj1 ~ ., 
                         data=training_SA_To_5_full[SA_To_5_folds[[1]],-c(30,31)])
# summary(Cox_SA_To_5_fld1)
# Rsquare= 0.665
S0_SA_To_5_fld1 = summary(survfit(Cox_SA_To_5_fld1, 
                                  newdata=training_SA_To_5_full[-SA_To_5_folds[[1]],-c(30,31)]), 
                          time=5)$surv
lp_SA_To_5_fld1 = predict(Cox_SA_To_5_fld1, 
                          newdata = training_SA_To_5_full[-SA_To_5_folds[[1]],-c(30,31)], type = "lp")
St_SA_To_5_fld1 = S0_SA_To_5_fld1^exp(lp_SA_To_5_fld1)
Acc_SA_To_5_fld1 = confusionMatrix(factor(training_SA_To_5_full[-SA_To_5_folds[[1]],]$status),
                                   factor(ifelse(St_SA_To_5_fld1>=0.5, 0 , 1)))$overall["Accuracy"]
Acc_SA_To_5_fld1
# Accuracy : 0.9189

# >>>>>>>>>>>>>>>>>
# 2nd fold
set.seed(123)
SA_To_5_obj2 = Surv(training_SA_To_5_full[SA_To_5_folds[[2]],]$time, 
                    training_SA_To_5_full[SA_To_5_folds[[2]],]$status)
Cox_SA_To_5_fld2 = coxph(SA_To_5_obj2 ~ ., 
                         data=training_SA_To_5_full[SA_To_5_folds[[2]],-c(30,31)])
# summary(Cox_SA_To_5_fld2)
# Rsquare= 0.667
S0_SA_To_5_fld2 = summary(survfit(Cox_SA_To_5_fld2, 
                                  newdata=training_SA_To_5_full[-SA_To_5_folds[[2]],-c(30,31)]), 
                          time=5)$surv
lp_SA_To_5_fld2 = predict(Cox_SA_To_5_fld2, 
                          newdata = training_SA_To_5_full[-SA_To_5_folds[[2]],-c(30,31)], type = "lp")
St_SA_To_5_fld2 = S0_SA_To_5_fld2^exp(lp_SA_To_5_fld2)
Acc_SA_To_5_fld2 = confusionMatrix(factor(training_SA_To_5_full[-SA_To_5_folds[[2]],]$status),
                                   factor(ifelse(St_SA_To_5_fld2>=0.5, 0 , 1)))$overall["Accuracy"]
Acc_SA_To_5_fld2
# Accuracy : 0.9444

# >>>>>>>>>>>>>>>>>
# 3rd fold
set.seed(123)
SA_To_5_obj3 = Surv(training_SA_To_5_full[SA_To_5_folds[[3]],]$time, 
                    training_SA_To_5_full[SA_To_5_folds[[3]],]$status)
Cox_SA_To_5_fld3 = coxph(SA_To_5_obj3 ~ ., 
                         data=training_SA_To_5_full[SA_To_5_folds[[3]],-c(30,31)])
# summary(Cox_SA_To_5_fld3)
# Rsquare= 0.668
S0_SA_To_5_fld3 = summary(survfit(Cox_SA_To_5_fld3, 
                                  newdata=training_SA_To_5_full[-SA_To_5_folds[[3]],-c(30,31)]), 
                          time=5)$surv
lp_SA_To_5_fld3 = predict(Cox_SA_To_5_fld3, 
                          newdata = training_SA_To_5_full[-SA_To_5_folds[[3]],-c(30,31)], type = "lp")
St_SA_To_5_fld3 = S0_SA_To_5_fld3^exp(lp_SA_To_5_fld3)
Acc_SA_To_5_fld3 = confusionMatrix(factor(training_SA_To_5_full[-SA_To_5_folds[[3]],]$status),
                                   factor(ifelse(St_SA_To_5_fld3>=0.5, 0 , 1)))$overall["Accuracy"]
Acc_SA_To_5_fld3
# Accuracy : 0.9189

# >>>>>>>>>>>>>>>>>
# 4th fold
set.seed(123)
SA_To_5_obj4 = Surv(training_SA_To_5_full[SA_To_5_folds[[4]],]$time, 
                    training_SA_To_5_full[SA_To_5_folds[[4]],]$status)
Cox_SA_To_5_fld4 = coxph(SA_To_5_obj4 ~ ., 
                         data=training_SA_To_5_full[SA_To_5_folds[[4]],-c(30,31)])
# summary(Cox_SA_To_5_fld4)
# Rsquare= 0.2682
S0_SA_To_5_fld4 = summary(survfit(Cox_SA_To_5_fld4, 
                                  newdata=training_SA_To_5_full[-SA_To_5_folds[[4]],-c(30,31)]), 
                          time=5)$surv
lp_SA_To_5_fld4 = predict(Cox_SA_To_5_fld4, 
                          newdata = training_SA_To_5_full[-SA_To_5_folds[[4]],-c(30,31)], type = "lp")
St_SA_To_5_fld4 = S0_SA_To_5_fld4^exp(lp_SA_To_5_fld4)
Acc_SA_To_5_fld4 = confusionMatrix(factor(training_SA_To_5_full[-SA_To_5_folds[[4]],]$status),
                                   factor(ifelse(St_SA_To_5_fld4>=0.5, 0 , 1)))$overall["Accuracy"]
Acc_SA_To_5_fld4
# Accuracy : 0.9189

# >>>>>>>>>>>>>>>>>
# 5th fold
set.seed(123)
SA_To_5_obj5 = Surv(training_SA_To_5_full[SA_To_5_folds[[5]],]$time, 
                    training_SA_To_5_full[SA_To_5_folds[[5]],]$status)
Cox_SA_To_5_fld5 = coxph(SA_To_5_obj5 ~ ., 
                         data=training_SA_To_5_full[SA_To_5_folds[[5]],-c(30,31)])
# summary(Cox_SA_To_5_fld5)
# Rsquare= 0.679
S0_SA_To_5_fld5 = summary(survfit(Cox_SA_To_5_fld5, 
                                  newdata=training_SA_To_5_full[-SA_To_5_folds[[5]],-c(30,31)]), 
                          time=5)$surv
lp_SA_To_5_fld5 = predict(Cox_SA_To_5_fld5, 
                          newdata = training_SA_To_5_full[-SA_To_5_folds[[5]],-c(30,31)], type = "lp")
St_SA_To_5_fld5 = S0_SA_To_5_fld5^exp(lp_SA_To_5_fld5)
Acc_SA_To_5_fld5 = confusionMatrix(factor(training_SA_To_5_full[-SA_To_5_folds[[5]],]$status),
                                   factor(ifelse(St_SA_To_5_fld5>=0.5, 0 , 1)))$overall["Accuracy"]
Acc_SA_To_5_fld5
# Accuracy : 0.8611

# >>>>>>>>>>>>>>>>>
# 6th fold
set.seed(123)
SA_To_5_obj6 = Surv(training_SA_To_5_full[SA_To_5_folds[[6]],]$time, 
                    training_SA_To_5_full[SA_To_5_folds[[6]],]$status)
Cox_SA_To_5_fld6 = coxph(SA_To_5_obj6 ~ ., 
                         data=training_SA_To_5_full[SA_To_5_folds[[6]],-c(30,31)])
# summary(Cox_SA_To_5_fld6)
# Rsquare= 0.673
S0_SA_To_5_fld6 = summary(survfit(Cox_SA_To_5_fld6, 
                                  newdata=training_SA_To_5_full[-SA_To_5_folds[[6]],-c(30,31)]), 
                          time=5)$surv
lp_SA_To_5_fld6 = predict(Cox_SA_To_5_fld6, 
                          newdata = training_SA_To_5_full[-SA_To_5_folds[[6]],-c(30,31)], type = "lp")
St_SA_To_5_fld6 = S0_SA_To_5_fld6^exp(lp_SA_To_5_fld6)
Acc_SA_To_5_fld6 = confusionMatrix(factor(training_SA_To_5_full[-SA_To_5_folds[[6]],]$status),
                                   factor(ifelse(St_SA_To_5_fld6>=0.5, 0 , 1)))$overall["Accuracy"]
Acc_SA_To_5_fld6
# Accuracy : 0.8649

# >>>>>>>>>>>>>>>>>
# 7th fold
set.seed(123)
SA_To_5_obj7 = Surv(training_SA_To_5_full[SA_To_5_folds[[7]],]$time, 
                    training_SA_To_5_full[SA_To_5_folds[[7]],]$status)
Cox_SA_To_5_fld7 = coxph(SA_To_5_obj7 ~ ., 
                         data=training_SA_To_5_full[SA_To_5_folds[[7]],-c(30,31)])
# summary(Cox_SA_To_5_fld7)
# Rsquare= 0.528
S0_SA_To_5_fld7 = summary(survfit(Cox_SA_To_5_fld7, 
                                  newdata=training_SA_To_5_full[-SA_To_5_folds[[7]],-c(30,31)]), 
                          time=5)$surv
lp_SA_To_5_fld7 = predict(Cox_SA_To_5_fld7, 
                          newdata = training_SA_To_5_full[-SA_To_5_folds[[7]],-c(30,31)], type = "lp")
St_SA_To_5_fld7 = S0_SA_To_5_fld7^exp(lp_SA_To_5_fld7)
Acc_SA_To_5_fld7 = confusionMatrix(factor(training_SA_To_5_full[-SA_To_5_folds[[7]],]$status),
                                   factor(ifelse(St_SA_To_5_fld7>=0.5, 0 , 1)))$overall["Accuracy"]
Acc_SA_To_5_fld7
# Accuracy : 0.8378

# >>>>>>>>>>>>>>>>>
# 8th fold
set.seed(123)
SA_To_5_obj8 = Surv(training_SA_To_5_full[SA_To_5_folds[[8]],]$time, 
                    training_SA_To_5_full[SA_To_5_folds[[8]],]$status)
Cox_SA_To_5_fld8 = coxph(SA_To_5_obj8 ~ ., 
                         data=training_SA_To_5_full[SA_To_5_folds[[8]],-c(30,31)])
# summary(Cox_SA_To_5_fld8)
# Rsquare= 0.678
S0_SA_To_5_fld8 = summary(survfit(Cox_SA_To_5_fld8, 
                                  newdata=training_SA_To_5_full[-SA_To_5_folds[[8]],-c(30,31)]), 
                          time=5)$surv
lp_SA_To_5_fld8 = predict(Cox_SA_To_5_fld8, 
                          newdata = training_SA_To_5_full[-SA_To_5_folds[[8]],-c(30,31)], type = "lp")
St_SA_To_5_fld8 = S0_SA_To_5_fld8^exp(lp_SA_To_5_fld8)
Acc_SA_To_5_fld8 = confusionMatrix(factor(training_SA_To_5_full[-SA_To_5_folds[[8]],]$status),
                                   factor(ifelse(St_SA_To_5_fld8>=0.5, 0 , 1)))$overall["Accuracy"]
Acc_SA_To_5_fld8
# Accuracy : 0.8649

# >>>>>>>>>>>>>>>>>
# 9th fold
set.seed(123)
SA_To_5_obj9 = Surv(training_SA_To_5_full[SA_To_5_folds[[9]],]$time, 
                    training_SA_To_5_full[SA_To_5_folds[[9]],]$status)
Cox_SA_To_5_fld9 = coxph(SA_To_5_obj9 ~ ., 
                         data=training_SA_To_5_full[SA_To_5_folds[[9]],-c(30,31)])
# summary(Cox_SA_To_5_fld9)
# Rsquare= 0.685
S0_SA_To_5_fld9 = summary(survfit(Cox_SA_To_5_fld9, 
                                  newdata=training_SA_To_5_full[-SA_To_5_folds[[9]],-c(30,31)]), 
                          time=5)$surv
lp_SA_To_5_fld9 = predict(Cox_SA_To_5_fld9, 
                          newdata = training_SA_To_5_full[-SA_To_5_folds[[9]],-c(30,31)], type = "lp")
St_SA_To_5_fld9 = S0_SA_To_5_fld9^exp(lp_SA_To_5_fld9)
Acc_SA_To_5_fld9 = confusionMatrix(factor(training_SA_To_5_full[-SA_To_5_folds[[9]],]$status),
                                   factor(ifelse(St_SA_To_5_fld9>=0.5, 0 , 1)))$overall["Accuracy"]
Acc_SA_To_5_fld9
# Accuracy : 0.8889

# >>>>>>>>>>>>>>>>>
# 10th fold
set.seed(123)
SA_To_5_obj10 = Surv(training_SA_To_5_full[SA_To_5_folds[[10]],]$time, 
                     training_SA_To_5_full[SA_To_5_folds[[10]],]$status)
Cox_SA_To_5_fld10 = coxph(SA_To_5_obj10 ~ ., 
                          data=training_SA_To_5_full[SA_To_5_folds[[10]],-c(30,31)])
# summary(Cox_SA_To_5_fld10)
# Rsquare= 0.674
S0_SA_To_5_fld10 = summary(survfit(Cox_SA_To_5_fld10, 
                                   newdata=training_SA_To_5_full[-SA_To_5_folds[[10]],-c(30,31)]), 
                           time=5)$surv
lp_SA_To_5_fld10 = predict(Cox_SA_To_5_fld10, 
                           newdata = training_SA_To_5_full[-SA_To_5_folds[[10]],-c(30,31)], type = "lp")
St_SA_To_5_fld10 = S0_SA_To_5_fld10^exp(lp_SA_To_5_fld10)
Acc_SA_To_5_fld10 = confusionMatrix(factor(training_SA_To_5_full[-SA_To_5_folds[[10]],]$status),
                                    factor(ifelse(St_SA_To_5_fld10>=0.5, 0 , 1)))$overall["Accuracy"]
Acc_SA_To_5_fld10
# Accuracy : 0.8649

# >>>>>>>>>>>>>>>>>
# Mean Accuracy
Acc_SA_To_5 = c(Acc_SA_To_5_fld1[[1]], Acc_SA_To_5_fld2[[1]], Acc_SA_To_5_fld3[[1]], 
                Acc_SA_To_5_fld4[[1]], Acc_SA_To_5_fld5[[1]], Acc_SA_To_5_fld6[[1]],
                Acc_SA_To_5_fld7[[1]], Acc_SA_To_5_fld8[[1]], Acc_SA_To_5_fld9[[1]], 
                Acc_SA_To_5_fld10[[1]])
(mean_Acc_SA_To_5 <- mean(Acc_SA_To_5))
# Accuracy : 0.8884

toc() # 1.16

########################################################################################
# Final Model
########################################################################################
# find out which training models returns the highest accuracy
which(Acc_SA_To_5==max(Acc_SA_To_5)) # 2

# find out significant predictors
summary(Cox_SA_To_5_fld2)
# ParcelArea+Wood_dist+lu4_dist+lu5_dist+lu2006+lc2006+lc2010+Change_AveIncome+F_lu7+F_lu9

# construct final model
set.seed(123)
SA_To_5_final.obj = Surv(training_SA_To_5_full[SA_To_5_folds[[2]],]$time, training_SA_To_5_full[SA_To_5_folds[[2]],]$status)
Cox_SA_To_5_final = coxph(SA_To_5_final.obj ~ ParcelArea+Wood_dist+lu4_dist+lu5_dist
                          +lu2006+lc2006+lc2010+Change_AveIncome+F_lu7+F_lu9, 
                          data=training_SA_To_5_full[SA_To_5_folds[[2]],-c(30,31)])
# summary(Cox_SA_To_5_final)
# Rsquare= 0.647
S0_SA_To_5_final = summary(survfit(Cox_SA_To_5_final, 
                                   newdata=testing_SA_To_5_full[,-c(30,31)]), time=5)$surv
lp_SA_To_5_final = predict(Cox_SA_To_5_final, 
                           newdata = testing_SA_To_5_full[,-c(30,31)], type = "lp")
St_SA_To_5_final = S0_SA_To_5_final^exp(lp_SA_To_5_final)
Acc_SA_To_5_final = confusionMatrix(factor(testing_SA_To_5_full$status),
                                    factor(ifelse(St_SA_To_5_final>=0.5, 0 , 1)))$overall["Accuracy"]
Acc_SA_To_5_final
# Accuracy : 0.9182

# remove ParcelArea, lu5_dist
Cox_SA_To_5_final2 = coxph(SA_To_5_final.obj ~ Wood_dist+lu4_dist
                           +lu2006+lc2006+lc2010+Change_AveIncome+F_lu7+F_lu9, 
                          data=training_SA_To_5_full[SA_To_5_folds[[2]],-c(30,31)])
# summary(Cox_SA_To_5_final2)
# Rsquare= 0.647
S0_SA_To_5_final2 = summary(survfit(Cox_SA_To_5_final2, 
                                   newdata=testing_SA_To_5_full[,-c(30,31)]), time=5)$surv
lp_SA_To_5_final2 = predict(Cox_SA_To_5_final2, 
                           newdata = testing_SA_To_5_full[,-c(30,31)], type = "lp")
St_SA_To_5_final2 = S0_SA_To_5_final2^exp(lp_SA_To_5_final2)
Acc_SA_To_5_final2 = confusionMatrix(factor(testing_SA_To_5_full$status),
                                    factor(ifelse(St_SA_To_5_final2>=0.5, 0 , 1)))$overall["Accuracy"]
Acc_SA_To_5_final2 # 0.8994

# remove Wood_dist
Cox_SA_To_5_final3 = coxph(SA_To_5_final.obj ~ lu4_dist
                           +Change_AveIncome+F_lu7+F_lu9 +lu2006+lc2006+lc2010, 
                           data=training_SA_To_5_full[SA_To_5_folds[[2]],-c(30,31)])
# summary(Cox_SA_To_5_final3)
# Rsquare= 0.64
S0_SA_To_5_final3 = summary(survfit(Cox_SA_To_5_final3, 
                                    newdata=testing_SA_To_5_full[,-c(30,31)]), time=5)$surv
lp_SA_To_5_final3 = predict(Cox_SA_To_5_final3, 
                            newdata = testing_SA_To_5_full[,-c(30,31)], type = "lp")
St_SA_To_5_final3 = S0_SA_To_5_final3^exp(lp_SA_To_5_final3)
Acc_SA_To_5_final3 = confusionMatrix(factor(testing_SA_To_5_full$status),
                                     factor(ifelse(St_SA_To_5_final3>=0.5, 0 , 1)))$overall["Accuracy"]
Acc_SA_To_5_final3 # 0.9119

anova(Cox_SA_To_5_final,Cox_SA_To_5_final2,test="Chisq") # final2
anova(Cox_SA_To_5_final2,Cox_SA_To_5_final3,test="Chisq") # final3
anova(Cox_SA_To_5_final,Cox_SA_To_5_final3,test="Chisq") # final3
# final3