# library(sperrorest)
# library(caret)
# library(mgcv)

training_To_11_full$lu2006 = factor(training_To_11_full$lu2006)
training_To_11_full$lc2010 = factor(training_To_11_full$lc2010)
training_To_11_full$lc2006 = factor(training_To_11_full$lc2006)
training_To_11_full$Response = factor(training_To_11_full$Response)

testing_To_11_full$lu2006 = factor(testing_To_11_full$lu2006)
testing_To_11_full$lc2010 = factor(testing_To_11_full$lc2010)
testing_To_11_full$lc2006 = factor(testing_To_11_full$lc2006)
testing_To_11_full$Response = factor(testing_To_11_full$Response)

training_To_11_full_data = data.frame(training_To_11_full[,-1])
testing_To_11_full_data = data.frame(testing_To_11_full[,-1])

training_To_11_full_data1.2 = training_To_11_full_data[,-c(2,3,27:30,34)]
testing_To_11_full_data1.2 = testing_To_11_full_data[,-c(2,3,27:30,34)]

############################################
# set up trainControl and grid
############################################
# using train() to conduct gam
ctrl = trainControl(method="none", savePredictions=TRUE)
grid = data.frame(method="GCV.Cp", select=FALSE)

############################################
# doing the spatial CV manually
############################################
set.seed(123)
GAM_To_11_full_folds = createFolds(training_To_11_full_data1.2$Response, k=10,list = TRUE, returnTrain = TRUE)

# fold 1
set.seed(123)
GAM_To_11_full_fld1 <- train(Response~., data=training_To_11_full_data1.2[GAM_To_11_full_folds[[1]],], 
                                method="gam", family="binomial",
                                trControl=ctrl, tuneGrid=grid, metric="Accuracy")
GAM_To_11_full_fld1_pred <- predict(GAM_To_11_full_fld1, newdata=training_To_11_full_data1.2[-GAM_To_11_full_folds[[1]],])
GAM_To_11_full_fld1_CM = confusionMatrix(factor(GAM_To_11_full_fld1_pred), training_To_11_full_data1.2[-GAM_To_11_full_folds[[1]],]$Response) 
(GAM_To_11_full_fld1_Acc = GAM_To_11_full_fld1_CM$overall[[1]])

# fold 2
set.seed(123)
GAM_To_11_full_fld2 <- train(Response~., data=training_To_11_full_data1.2[GAM_To_11_full_folds[[2]],], 
                                method="gam", family="binomial",
                                trControl=ctrl, tuneGrid=grid, metric="Accuracy")
GAM_To_11_full_fld2_pred <- predict(GAM_To_11_full_fld2, newdata=training_To_11_full_data1.2[-GAM_To_11_full_folds[[2]],])
GAM_To_11_full_fld2_CM = confusionMatrix(factor(GAM_To_11_full_fld2_pred), training_To_11_full_data1.2[-GAM_To_11_full_folds[[2]],]$Response)
(GAM_To_11_full_fld2_Acc = GAM_To_11_full_fld2_CM$overall[[1]])

# fold 3
set.seed(123)
GAM_To_11_full_fld3 <- train(Response~., data=training_To_11_full_data1.2[GAM_To_11_full_folds[[3]],], 
                                method="gam", family="binomial",
                                trControl=ctrl, tuneGrid=grid, metric="Accuracy")
GAM_To_11_full_fld3_pred <- predict(GAM_To_11_full_fld3, newdata=training_To_11_full_data1.2[-GAM_To_11_full_folds[[3]],])
GAM_To_11_full_fld3_CM = confusionMatrix(factor(GAM_To_11_full_fld3_pred), training_To_11_full_data1.2[-GAM_To_11_full_folds[[3]],]$Response) 
(GAM_To_11_full_fld3_Acc = GAM_To_11_full_fld3_CM$overall[[1]])

# fold 4
set.seed(123)
GAM_To_11_full_fld4 <- train(Response~., data=training_To_11_full_data1.2[GAM_To_11_full_folds[[4]],], 
                                method="gam", family="binomial",
                                trControl=ctrl, tuneGrid=grid, metric="Accuracy")
GAM_To_11_full_fld4_pred <- predict(GAM_To_11_full_fld4, newdata=training_To_11_full_data1.2[-GAM_To_11_full_folds[[4]],])
GAM_To_11_full_fld4_CM = confusionMatrix(factor(GAM_To_11_full_fld4_pred), training_To_11_full_data1.2[-GAM_To_11_full_folds[[4]],]$Response)
(GAM_To_11_full_fld4_Acc = GAM_To_11_full_fld4_CM$overall[[1]])

# fold 5
set.seed(123)
GAM_To_11_full_fld5 <- train(Response~., data=training_To_11_full_data1.2[GAM_To_11_full_folds[[5]],], 
                                method="gam", family="binomial",
                                trControl=ctrl, tuneGrid=grid, metric="Accuracy")
GAM_To_11_full_fld5_pred <- predict(GAM_To_11_full_fld5, newdata=training_To_11_full_data1.2[-GAM_To_11_full_folds[[5]],])
GAM_To_11_full_fld5_CM = confusionMatrix(factor(GAM_To_11_full_fld5_pred), training_To_11_full_data1.2[-GAM_To_11_full_folds[[5]],]$Response)
(GAM_To_11_full_fld5_Acc = GAM_To_11_full_fld5_CM$overall[[1]])

# fold 6
set.seed(123)
GAM_To_11_full_fld6 <- train(Response~., data=training_To_11_full_data1.2[GAM_To_11_full_folds[[6]],], 
                                method="gam", family="binomial",
                                trControl=ctrl, tuneGrid=grid, metric="Accuracy")
GAM_To_11_full_fld6_pred <- predict(GAM_To_11_full_fld6, newdata=training_To_11_full_data1.2[-GAM_To_11_full_folds[[6]],])
GAM_To_11_full_fld6_CM = confusionMatrix(factor(GAM_To_11_full_fld6_pred), training_To_11_full_data1.2[-GAM_To_11_full_folds[[6]],]$Response) 
(GAM_To_11_full_fld6_Acc = GAM_To_11_full_fld6_CM$overall[[1]])

# fold 7
set.seed(123)
GAM_To_11_full_fld7 <- train(Response~., data=training_To_11_full_data1.2[GAM_To_11_full_folds[[7]],], 
                                method="gam", family="binomial",
                                trControl=ctrl, tuneGrid=grid, metric="Accuracy")
GAM_To_11_full_fld7_pred <- predict(GAM_To_11_full_fld7, newdata=training_To_11_full_data1.2[-GAM_To_11_full_folds[[7]],])
GAM_To_11_full_fld7_CM = confusionMatrix(factor(GAM_To_11_full_fld7_pred), training_To_11_full_data1.2[-GAM_To_11_full_folds[[7]],]$Response) 
(GAM_To_11_full_fld7_Acc = GAM_To_11_full_fld7_CM$overall[[1]])

# fold 8
set.seed(123)
GAM_To_11_full_fld8 <- train(Response~., data=training_To_11_full_data1.2[GAM_To_11_full_folds[[8]],], 
                                method="gam", family="binomial",
                                trControl=ctrl, tuneGrid=grid, metric="Accuracy")
GAM_To_11_full_fld8_pred <- predict(GAM_To_11_full_fld8, newdata=training_To_11_full_data1.2[-GAM_To_11_full_folds[[8]],])
GAM_To_11_full_fld8_CM = confusionMatrix(factor(GAM_To_11_full_fld8_pred), training_To_11_full_data1.2[-GAM_To_11_full_folds[[8]],]$Response) 
(GAM_To_11_full_fld8_Acc = GAM_To_11_full_fld8_CM$overall[[1]])

# fold 9
set.seed(123)
GAMTo_11_full_fld9 <- train(Response~., data=training_To_11_full_data1.2[GAM_To_11_full_folds[[9]],], 
                               method="gam", family="binomial",
                               trControl=ctrl, tuneGrid=grid, metric="Accuracy")
GAMTo_11_full_fld9_pred <- predict(GAMTo_11_full_fld9, newdata=training_To_11_full_data1.2[-GAM_To_11_full_folds[[9]],])
GAMTo_11_full_fld9_CM = confusionMatrix(factor(GAMTo_11_full_fld9_pred), training_To_11_full_data1.2[-GAM_To_11_full_folds[[9]],]$Response) 
(GAMTo_11_full_fld9_Acc = GAMTo_11_full_fld9_CM$overall[[1]])

# fold 10
set.seed(123)
GAM_To_11_full_fld10 <- train(Response~., data=training_To_11_full_data1.2[GAM_To_11_full_folds[[10]],], 
                                 method="gam", family="binomial",
                                 trControl=ctrl, tuneGrid=grid, metric="Accuracy")
GAM_To_11_full_fld10_pred <- predict(GAM_To_11_full_fld10, newdata=training_To_11_full_data1.2[-GAM_To_11_full_folds[[10]],])
GAM_To_11_full_fld10_CM = confusionMatrix(factor(GAM_To_11_full_fld10_pred), training_To_11_full_data1.2[-GAM_To_11_full_folds[[10]],]$Response) 
(GAM_To_11_full_fld10_Acc = GAM_To_11_full_fld10_CM$overall[[1]])

GAM_To_11_full_Acc = c(GAM_To_11_full_fld1_Acc, GAM_To_11_full_fld2_Acc, GAM_To_11_full_fld3_Acc, 
                          GAM_To_11_full_fld4_Acc, GAM_To_11_full_fld5_Acc, GAM_To_11_full_fld6_Acc,
                          GAM_To_11_full_fld7_Acc, GAM_To_11_full_fld8_Acc, GAMTo_11_full_fld9_Acc, GAM_To_11_full_fld10_Acc)
(GAM_To_11_full_meanAcc = mean(GAM_To_11_full_Acc))
# 0.8766243

############################################
# building final models
############################################
which(GAM_To_11_full_Acc == max(GAM_To_11_full_Acc)) # 3
# choose GAM model with fold 3
summary(GAM_To_11_full_fld3)
# significant predictors: 
# lu2006 + lc2010 + s(F_lu2) + s(Residential_Popn_Density)
# + s(Change_Popn) + s(MeanSlope) + s(lu4_dist) + s(lu8_dist) + s(lu9_dist)
# + s(MeanDEM) + s(Wood_dist) + s(ParcelArea) + s(Ramp_dist) + s(River_dist) + s(MRoad_dist)

set.seed(123)
GAM_To_11_full_final = gam(Response ~ lu2006 + lc2010 + s(F_lu2) + s(Residential_Popn_Density)
                           + s(Change_Popn) + s(MeanSlope) + s(lu4_dist) + s(lu8_dist) + s(lu9_dist)
                           + s(MeanDEM) + s(Wood_dist) + s(ParcelArea) + s(Ramp_dist) + s(River_dist) + s(MRoad_dist), 
                           data = training_To_11_full_data1.2[GAM_To_11_full_folds[[3]],], family=binomial(link=logit))
summary(GAM_To_11_full_final)

# use regular MeanSlope
GAM_To_11_full_final2 = gam(Response ~ lu2006 + lc2010 + s(F_lu2) + s(Residential_Popn_Density)
                            + s(Change_Popn) + MeanSlope + s(lu4_dist) + s(lu8_dist) + s(lu9_dist)
                            + s(MeanDEM) + s(Wood_dist) + s(ParcelArea) + s(Ramp_dist) + s(River_dist) + s(MRoad_dist), 
                            data = training_To_11_full_data1.2[GAM_To_11_full_folds[[3]],],  family=binomial(link=logit))
summary(GAM_To_11_full_final2)

GAM_To_11_full_final2_pred <- predict(GAM_To_11_full_final2, newdata=testing_To_11_full_data1.2, type = "response")
GAM_To_11_full_final2_pred1.1 <- GAM_To_11_full_final2_pred
GAM_To_11_full_final2_pred1.1[which(GAM_To_11_full_final2_pred<0.5)] <- 0
GAM_To_11_full_final2_pred1.1[which(GAM_To_11_full_final2_pred>=0.5)] <- 1
confusionMatrix(factor(GAM_To_11_full_final2_pred1.1), testing_To_11_full_data1.2$Response) 
# Accuracy: 0.8587

