############################################
# set up trainControl and grid
############################################
# using train() to conduct gam
ctrl = trainControl(method="none", savePredictions=TRUE)
grid = data.frame(method="GCV.Cp", select=FALSE)

############################################
# doing the spatial CV manually
############################################
set.seed(123)
GAM_To_4_reduced_folds = createFolds(training_To_4_reduced_data1.2$Response, k=10,list = TRUE, returnTrain = TRUE)


# fold 1
set.seed(123)
GAM_To_4_reduced_fld1 <- train(Response~., data=training_To_4_reduced_data1.2[GAM_To_4_reduced_folds[[1]],], 
                            method="gam", family="binomial",
                            trControl=ctrl, tuneGrid=grid, metric="Accuracy")
GAM_To_4_reduced_fld1_pred <- predict(GAM_To_4_reduced_fld1, newdata=training_To_4_reduced_data1.2[-GAM_To_4_reduced_folds[[1]],])
GAM_To_4_reduced_fld1_CM = confusionMatrix(factor(GAM_To_4_reduced_fld1_pred), training_To_4_reduced_data1.2[-GAM_To_4_reduced_folds[[1]],]$Response) 
(GAM_To_4_reduced_fld1_Acc = GAM_To_4_reduced_fld1_CM$overall[[1]])

# fold 2
set.seed(123)
GAM_To_4_reduced_fld2 <- train(Response~., data=training_To_4_reduced_data1.2[GAM_To_4_reduced_folds[[2]],], 
                            method="gam", family="binomial",
                            trControl=ctrl, tuneGrid=grid, metric="Accuracy")
GAM_To_4_reduced_fld2_pred <- predict(GAM_To_4_reduced_fld2, newdata=training_To_4_reduced_data1.2[-GAM_To_4_reduced_folds[[2]],])
GAM_To_4_reduced_fld2_CM = confusionMatrix(factor(GAM_To_4_reduced_fld2_pred), training_To_4_reduced_data1.2[-GAM_To_4_reduced_folds[[2]],]$Response)
(GAM_To_4_reduced_fld2_Acc = GAM_To_4_reduced_fld2_CM$overall[[1]])

# fold 3
set.seed(123)
GAM_To_4_reduced_fld3 <- train(Response~., data=training_To_4_reduced_data1.2[GAM_To_4_reduced_folds[[3]],], 
                            method="gam", family="binomial",
                            trControl=ctrl, tuneGrid=grid, metric="Accuracy")
GAM_To_4_reduced_fld3_pred <- predict(GAM_To_4_reduced_fld3, newdata=training_To_4_reduced_data1.2[-GAM_To_4_reduced_folds[[3]],])
GAM_To_4_reduced_fld3_CM = confusionMatrix(factor(GAM_To_4_reduced_fld3_pred), training_To_4_reduced_data1.2[-GAM_To_4_reduced_folds[[3]],]$Response) 
(GAM_To_4_reduced_fld3_Acc = GAM_To_4_reduced_fld3_CM$overall[[1]])

# fold 4
set.seed(123)
GAM_To_4_reduced_fld4 <- train(Response~., data=training_To_4_reduced_data1.2[GAM_To_4_reduced_folds[[4]],], 
                            method="gam", family="binomial",
                            trControl=ctrl, tuneGrid=grid, metric="Accuracy")
GAM_To_4_reduced_fld4_pred <- predict(GAM_To_4_reduced_fld4, newdata=training_To_4_reduced_data1.2[-GAM_To_4_reduced_folds[[4]],])
GAM_To_4_reduced_fld4_CM = confusionMatrix(factor(GAM_To_4_reduced_fld4_pred), training_To_4_reduced_data1.2[-GAM_To_4_reduced_folds[[4]],]$Response)
(GAM_To_4_reduced_fld4_Acc = GAM_To_4_reduced_fld4_CM$overall[[1]])

# fold 5
set.seed(123)
GAM_To_4_reduced_fld5 <- train(Response~., data=training_To_4_reduced_data1.2[GAM_To_4_reduced_folds[[5]],], 
                            method="gam", family="binomial",
                            trControl=ctrl, tuneGrid=grid, metric="Accuracy")
GAM_To_4_reduced_fld5_pred <- predict(GAM_To_4_reduced_fld5, newdata=training_To_4_reduced_data1.2[-GAM_To_4_reduced_folds[[5]],])
GAM_To_4_reduced_fld5_CM = confusionMatrix(factor(GAM_To_4_reduced_fld5_pred), training_To_4_reduced_data1.2[-GAM_To_4_reduced_folds[[5]],]$Response)
(GAM_To_4_reduced_fld5_Acc = GAM_To_4_reduced_fld5_CM$overall[[1]])

# fold 6
set.seed(123)
GAM_To_4_reduced_fld6 <- train(Response~., data=training_To_4_reduced_data1.2[GAM_To_4_reduced_folds[[6]],], 
                            method="gam", family="binomial",
                            trControl=ctrl, tuneGrid=grid, metric="Accuracy")
GAM_To_4_reduced_fld6_pred <- predict(GAM_To_4_reduced_fld6, newdata=training_To_4_reduced_data1.2[-GAM_To_4_reduced_folds[[6]],])
GAM_To_4_reduced_fld6_CM = confusionMatrix(factor(GAM_To_4_reduced_fld6_pred), training_To_4_reduced_data1.2[-GAM_To_4_reduced_folds[[6]],]$Response) 
(GAM_To_4_reduced_fld6_Acc = GAM_To_4_reduced_fld6_CM$overall[[1]])

# fold 7
set.seed(123)
GAM_To_4_reduced_fld7 <- train(Response~., data=training_To_4_reduced_data1.2[GAM_To_4_reduced_folds[[7]],], 
                            method="gam", family="binomial",
                            trControl=ctrl, tuneGrid=grid, metric="Accuracy")
GAM_To_4_reduced_fld7_pred <- predict(GAM_To_4_reduced_fld7, newdata=training_To_4_reduced_data1.2[-GAM_To_4_reduced_folds[[7]],])
GAM_To_4_reduced_fld7_CM = confusionMatrix(factor(GAM_To_4_reduced_fld7_pred), training_To_4_reduced_data1.2[-GAM_To_4_reduced_folds[[7]],]$Response) 
(GAM_To_4_reduced_fld7_Acc = GAM_To_4_reduced_fld7_CM$overall[[1]])

# fold 8
set.seed(123)
GAM_To_4_reduced_fld8 <- train(Response~., data=training_To_4_reduced_data1.2[GAM_To_4_reduced_folds[[8]],], 
                            method="gam", family="binomial",
                            trControl=ctrl, tuneGrid=grid, metric="Accuracy")
GAM_To_4_reduced_fld8_pred <- predict(GAM_To_4_reduced_fld8, newdata=training_To_4_reduced_data1.2[-GAM_To_4_reduced_folds[[8]],])
GAM_To_4_reduced_fld8_CM = confusionMatrix(factor(GAM_To_4_reduced_fld8_pred), training_To_4_reduced_data1.2[-GAM_To_4_reduced_folds[[8]],]$Response) 
(GAM_To_4_reduced_fld8_Acc = GAM_To_4_reduced_fld8_CM$overall[[1]])

# fold 9
set.seed(123)
GAMTo_4_reduced_fld9 <- train(Response~., data=training_To_4_reduced_data1.2[GAM_To_4_reduced_folds[[9]],], 
                           method="gam", family="binomial",
                           trControl=ctrl, tuneGrid=grid, metric="Accuracy")
GAMTo_4_reduced_fld9_pred <- predict(GAMTo_4_reduced_fld9, newdata=training_To_4_reduced_data1.2[-GAM_To_4_reduced_folds[[9]],])
GAMTo_4_reduced_fld9_CM = confusionMatrix(factor(GAMTo_4_reduced_fld9_pred), training_To_4_reduced_data1.2[-GAM_To_4_reduced_folds[[9]],]$Response) 
(GAMTo_4_reduced_fld9_Acc = GAMTo_4_reduced_fld9_CM$overall[[1]])

# fold 10
set.seed(123)
GAM_To_4_reduced_fld10 <- train(Response~., data=training_To_4_reduced_data1.2[GAM_To_4_reduced_folds[[10]],], 
                             method="gam", family="binomial",
                             trControl=ctrl, tuneGrid=grid, metric="Accuracy")
GAM_To_4_reduced_fld10_pred <- predict(GAM_To_4_reduced_fld10, newdata=training_To_4_reduced_data1.2[-GAM_To_4_reduced_folds[[10]],])
GAM_To_4_reduced_fld10_CM = confusionMatrix(factor(GAM_To_4_reduced_fld10_pred), training_To_4_reduced_data1.2[-GAM_To_4_reduced_folds[[10]],]$Response) 
(GAM_To_4_reduced_fld10_Acc = GAM_To_4_reduced_fld10_CM$overall[[1]])

GAM_To_4_reduced_Acc = c(GAM_To_4_reduced_fld1_Acc, GAM_To_4_reduced_fld2_Acc, GAM_To_4_reduced_fld3_Acc, 
                      GAM_To_4_reduced_fld4_Acc, GAM_To_4_reduced_fld5_Acc, GAM_To_4_reduced_fld6_Acc,
                      GAM_To_4_reduced_fld7_Acc, GAM_To_4_reduced_fld8_Acc, GAMTo_4_reduced_fld9_Acc, GAM_To_4_reduced_fld10_Acc)
(GAM_To_4_reduced_meanAcc = mean(GAM_To_4_reduced_Acc))
# 0.865301

############################################
# building final models
############################################
which(GAM_To_4_reduced_Acc == max(GAM_To_4_reduced_Acc)) # 7
# choose GAM model with fold 7
summary(GAM_To_4_reduced_fld7)
# significant predictors: 
# lu2006 + lc2006 + lc2010 
# +s(Change_AveIncome)+s(lu4_dist)+s(lu5_dist)+s(Wood_dist)+s(lu9_dist)
# +s(Water_dist)+s(ParcelArea)+s(Ramp_dist)+s(LRoad_dist)+s(MRoad_dist) 

GAM_To_4_reduced_final=gam(Response ~ lc2006+lu2006+lc2010
                              +s(ParcelArea)+s(Water_dist)+LRoad_dist+s(MRoad_dist)+s(lu4_dist)+s(lu8_dist), 
                              data=training_To_4_reduced_data1.2[GAM_To_4_reduced_folds[[7]],],
                              family=binomial(link=logit))
GAM_To_4_reduced_final_pred=predict(GAM_To_4_reduced_final,
                                    newdata=testing_To_4_reduced_data1.2[-c(which(testing_To_4_reduced_data1.2$lc2010==6),
                                                                            which(testing_To_4_reduced_data1.2$lu2006==10),
                                                                            which(testing_To_4_reduced_data1.2$lc2006==6)),], 
                                    type = "response")
GAM_To_4_reduced_final_pred[which(GAM_To_4_reduced_final_pred<0.5)] <- 0
GAM_To_4_reduced_final_pred[which(GAM_To_4_reduced_final_pred>=0.5)] <- 1
confusionMatrix(factor(GAM_To_4_reduced_final_pred), 
                testing_To_4_reduced_data1.2[-c(which(testing_To_4_reduced_data1.2$lc2010==6),
                                                which(testing_To_4_reduced_data1.2$lu2006==10),
                                                which(testing_To_4_reduced_data1.2$lc2006==6)),]$Response) 
# 0.8983
summary(GAM_To_4_reduced_final)

# remove lc2006
GAM_To_4_reduced_final2=gam(Response ~ lu2006+lc2010
                            +s(ParcelArea)+s(Water_dist)+LRoad_dist+s(MRoad_dist)+s(lu4_dist)+s(lu8_dist), 
                           data=training_To_4_reduced_data1.2[GAM_To_4_reduced_folds[[7]],],
                           family=binomial(link=logit))
GAM_To_4_reduced_final2_pred=predict(GAM_To_4_reduced_final2,
                                     newdata=testing_To_4_reduced_data1.2[-c(which(testing_To_4_reduced_data1.2$lc2010==6),
                                                                            which(testing_To_4_reduced_data1.2$lu2006==10),
                                                                            which(testing_To_4_reduced_data1.2$lc2006==6)),], 
                                    type = "response")
GAM_To_4_reduced_final2_pred[which(GAM_To_4_reduced_final2_pred<0.5)] <- 0
GAM_To_4_reduced_final2_pred[which(GAM_To_4_reduced_final2_pred>=0.5)] <- 1
confusionMatrix(factor(GAM_To_4_reduced_final2_pred), 
                testing_To_4_reduced_data1.2[-c(which(testing_To_4_reduced_data1.2$lc2010==6),
                                                which(testing_To_4_reduced_data1.2$lu2006==10),
                                                which(testing_To_4_reduced_data1.2$lc2006==6)),]$Response) 
# 0.9017
summary(GAM_To_4_reduced_final2)

anova(GAM_To_4_reduced_final,GAM_To_4_reduced_final2,test="Chisq")
# prefer final2