# library(sperrorest)
# library(caret)
# library(mgcv)

training_To_3_full$lu2006 = factor(training_To_3_full$lu2006)
training_To_3_full$lc2010 = factor(training_To_3_full$lc2010)
training_To_3_full$lc2006 = factor(training_To_3_full$lc2006)
training_To_3_full$Response = factor(training_To_3_full$Response)

testing_To_3_full$lu2006 = factor(testing_To_3_full$lu2006)
testing_To_3_full$lc2010 = factor(testing_To_3_full$lc2010)
testing_To_3_full$lc2006 = factor(testing_To_3_full$lc2006)
testing_To_3_full$Response = factor(testing_To_3_full$Response)

training_To_3_full_data = data.frame(training_To_3_full[,-1])
testing_To_3_full_data = data.frame(testing_To_3_full[,-1])

training_To_3_full_data1.2 = training_To_3_full_data[,-c(2,3,27,29,30,34)]
testing_To_3_full_data1.2 = testing_To_3_full_data[,-c(2,3,27,29,30,34)]

############################################
# set up trainControl and grid
############################################
# using train() to conduct gam
ctrl = trainControl(method="none", savePredictions=TRUE)
grid = data.frame(method="GCV.Cp", select=FALSE)

############################################
# doing the spatial CV manually
############################################
set.seed(123)
GAM_To_3_full_folds = createFolds(training_To_3_full_data1.2$Response, k=10,list = TRUE, returnTrain = TRUE)


# fold 1
set.seed(123)
GAM_To_3_full_fld1 <- train(Response~., data=training_To_3_full_data1.2[GAM_To_3_full_folds[[1]],], 
                            method="gam", family="binomial",
                            trControl=ctrl, tuneGrid=grid, metric="Accuracy")
GAM_To_3_full_fld1_pred <- predict(GAM_To_3_full_fld1, newdata=training_To_3_full_data1.2[-GAM_To_3_full_folds[[1]],])
GAM_To_3_full_fld1_CM = confusionMatrix(factor(GAM_To_3_full_fld1_pred), training_To_3_full_data1.2[-GAM_To_3_full_folds[[1]],]$Response) 
(GAM_To_3_full_fld1_Acc = GAM_To_3_full_fld1_CM$overall[[1]])

# fold 2
set.seed(123)
GAM_To_3_full_fld2 <- train(Response~., data=training_To_3_full_data1.2[GAM_To_3_full_folds[[2]],], 
                            method="gam", family="binomial",
                            trControl=ctrl, tuneGrid=grid, metric="Accuracy")
GAM_To_3_full_fld2_pred <- predict(GAM_To_3_full_fld2, newdata=training_To_3_full_data1.2[-GAM_To_3_full_folds[[2]],])
GAM_To_3_full_fld2_CM = confusionMatrix(factor(GAM_To_3_full_fld2_pred), training_To_3_full_data1.2[-GAM_To_3_full_folds[[2]],]$Response)
(GAM_To_3_full_fld2_Acc = GAM_To_3_full_fld2_CM$overall[[1]])

# fold 3
set.seed(123)
GAM_To_3_full_fld3 <- train(Response~., data=training_To_3_full_data1.2[GAM_To_3_full_folds[[3]],], 
                            method="gam", family="binomial",
                            trControl=ctrl, tuneGrid=grid, metric="Accuracy")
GAM_To_3_full_fld3_pred <- predict(GAM_To_3_full_fld3, newdata=training_To_3_full_data1.2[-GAM_To_3_full_folds[[3]],])
GAM_To_3_full_fld3_CM = confusionMatrix(factor(GAM_To_3_full_fld3_pred), training_To_3_full_data1.2[-GAM_To_3_full_folds[[3]],]$Response) 
(GAM_To_3_full_fld3_Acc = GAM_To_3_full_fld3_CM$overall[[1]])

# fold 4
set.seed(123)
GAM_To_3_full_fld4 <- train(Response~., data=training_To_3_full_data1.2[GAM_To_3_full_folds[[4]],], 
                            method="gam", family="binomial",
                            trControl=ctrl, tuneGrid=grid, metric="Accuracy")
GAM_To_3_full_fld4_pred <- predict(GAM_To_3_full_fld4, newdata=training_To_3_full_data1.2[-GAM_To_3_full_folds[[4]],])
GAM_To_3_full_fld4_CM = confusionMatrix(factor(GAM_To_3_full_fld4_pred), training_To_3_full_data1.2[-GAM_To_3_full_folds[[4]],]$Response)
(GAM_To_3_full_fld4_Acc = GAM_To_3_full_fld4_CM$overall[[1]])

# fold 5
set.seed(123)
GAM_To_3_full_fld5 <- train(Response~., data=training_To_3_full_data1.2[GAM_To_3_full_folds[[5]],], 
                            method="gam", family="binomial",
                            trControl=ctrl, tuneGrid=grid, metric="Accuracy")
GAM_To_3_full_fld5_pred <- predict(GAM_To_3_full_fld5, newdata=training_To_3_full_data1.2[-GAM_To_3_full_folds[[5]],])
GAM_To_3_full_fld5_CM = confusionMatrix(factor(GAM_To_3_full_fld5_pred), training_To_3_full_data1.2[-GAM_To_3_full_folds[[5]],]$Response)
(GAM_To_3_full_fld5_Acc = GAM_To_3_full_fld5_CM$overall[[1]])

# fold 6
set.seed(123)
GAM_To_3_full_fld6 <- train(Response~., data=training_To_3_full_data1.2[GAM_To_3_full_folds[[6]],], 
                            method="gam", family="binomial",
                            trControl=ctrl, tuneGrid=grid, metric="Accuracy")
GAM_To_3_full_fld6_pred <- predict(GAM_To_3_full_fld6, newdata=training_To_3_full_data1.2[-GAM_To_3_full_folds[[6]],])
GAM_To_3_full_fld6_CM = confusionMatrix(factor(GAM_To_3_full_fld6_pred), training_To_3_full_data1.2[-GAM_To_3_full_folds[[6]],]$Response) 
(GAM_To_3_full_fld6_Acc = GAM_To_3_full_fld6_CM$overall[[1]])

# fold 7
set.seed(123)
GAM_To_3_full_fld7 <- train(Response~., data=training_To_3_full_data1.2[GAM_To_3_full_folds[[7]],], 
                            method="gam", family="binomial",
                            trControl=ctrl, tuneGrid=grid, metric="Accuracy")
GAM_To_3_full_fld7_pred <- predict(GAM_To_3_full_fld7, newdata=training_To_3_full_data1.2[-GAM_To_3_full_folds[[7]],])
GAM_To_3_full_fld7_CM = confusionMatrix(factor(GAM_To_3_full_fld7_pred), training_To_3_full_data1.2[-GAM_To_3_full_folds[[7]],]$Response) 
(GAM_To_3_full_fld7_Acc = GAM_To_3_full_fld7_CM$overall[[1]])

# fold 8
set.seed(123)
GAM_To_3_full_fld8 <- train(Response~., data=training_To_3_full_data1.2[GAM_To_3_full_folds[[8]],], 
                            method="gam", family="binomial",
                            trControl=ctrl, tuneGrid=grid, metric="Accuracy")
GAM_To_3_full_fld8_pred <- predict(GAM_To_3_full_fld8, newdata=training_To_3_full_data1.2[-GAM_To_3_full_folds[[8]],])
GAM_To_3_full_fld8_CM = confusionMatrix(factor(GAM_To_3_full_fld8_pred), training_To_3_full_data1.2[-GAM_To_3_full_folds[[8]],]$Response) 
(GAM_To_3_full_fld8_Acc = GAM_To_3_full_fld8_CM$overall[[1]])

# fold 9
set.seed(123)
GAMTo_3_full_fld9 <- train(Response~., data=training_To_3_full_data1.2[GAM_To_3_full_folds[[9]],], 
                           method="gam", family="binomial",
                           trControl=ctrl, tuneGrid=grid, metric="Accuracy")
GAMTo_3_full_fld9_pred <- predict(GAMTo_3_full_fld9, newdata=training_To_3_full_data1.2[-GAM_To_3_full_folds[[9]],])
GAMTo_3_full_fld9_CM = confusionMatrix(factor(GAMTo_3_full_fld9_pred), training_To_3_full_data1.2[-GAM_To_3_full_folds[[9]],]$Response) 
(GAMTo_3_full_fld9_Acc = GAMTo_3_full_fld9_CM$overall[[1]])

# fold 10
set.seed(123)
GAM_To_3_full_fld10 <- train(Response~., data=training_To_3_full_data1.2[GAM_To_3_full_folds[[10]],], 
                             method="gam", family="binomial",
                             trControl=ctrl, tuneGrid=grid, metric="Accuracy")
GAM_To_3_full_fld10_pred <- predict(GAM_To_3_full_fld10, newdata=training_To_3_full_data1.2[-GAM_To_3_full_folds[[10]],])
GAM_To_3_full_fld10_CM = confusionMatrix(factor(GAM_To_3_full_fld10_pred), training_To_3_full_data1.2[-GAM_To_3_full_folds[[10]],]$Response) 
(GAM_To_3_full_fld10_Acc = GAM_To_3_full_fld10_CM$overall[[1]])

GAM_To_3_full_Acc = c(GAM_To_3_full_fld1_Acc, GAM_To_3_full_fld2_Acc, GAM_To_3_full_fld3_Acc, 
                      GAM_To_3_full_fld4_Acc, GAM_To_3_full_fld5_Acc, GAM_To_3_full_fld6_Acc,
                      GAM_To_3_full_fld7_Acc, GAM_To_3_full_fld8_Acc, GAMTo_3_full_fld9_Acc, GAM_To_3_full_fld10_Acc)
(GAM_To_3_full_meanAcc = mean(GAM_To_3_full_Acc))
# 0.7946

############################################
# building final models
############################################
which(GAM_To_3_full_Acc == max(GAM_To_3_full_Acc)) # 4
# choose GAM model with fold 4
summary(GAM_To_3_full_fld4)
# significant predictors: 
# lu2006 + lc2006 + lc2010 + ParcelArea
# + s(Residential_Popn_Density) + s(DA_Popn_Density) + s(Change_AveIncome) 
# + s(lu4_dist) + s(Wood_dist) + s(lu9_dist)

GAM_To_3_full_fld4_1.1=gam(Response ~ lu2006 + lc2006 + lc2010 + ParcelArea
                           + s(Residential_Popn_Density) + s(DA_Popn_Density) + s(Change_AveIncome) 
                           + s(lu4_dist) + s(Wood_dist) + s(lu9_dist), 
                           data=training_To_3_full_data1.2[GAM_To_3_full_folds[[4]],], family=binomial(link=logit))
GAM_To_3_full_fld4_1.1_pred=predict(GAM_To_3_full_fld4_1.1,newdata=testing_To_3_full_data1.2, type = "response")
GAM_To_3_full_fld4_1.1_pred[which(GAM_To_3_full_fld4_1.1_pred<0.5)] <- 0
GAM_To_3_full_fld4_1.1_pred[which(GAM_To_3_full_fld4_1.1_pred>=0.5)] <- 1
confusionMatrix(factor(GAM_To_3_full_fld4_1.1_pred), testing_To_3_full_data1.2$Response) # 0.7824
summary(GAM_To_3_full_fld2_1.1)

# remove lc2006
GAM_To_3_full_fld4_1.2=gam(Response ~ lu2006 + lc2010 + ParcelArea
                           + s(Residential_Popn_Density) + s(DA_Popn_Density) + s(Change_AveIncome) 
                           + s(lu4_dist) + s(Wood_dist) + s(lu9_dist), 
                           data=training_To_3_full_data1.2[GAM_To_3_full_folds[[4]],], family=binomial(link=logit))
GAM_To_3_full_fld4_1.2_pred=predict(GAM_To_3_full_fld4_1.2,newdata=testing_To_3_full_data1.2, type = "response")
GAM_To_3_full_fld4_1.2_pred[which(GAM_To_3_full_fld4_1.2_pred<0.5)] <- 0
GAM_To_3_full_fld4_1.2_pred[which(GAM_To_3_full_fld4_1.2_pred>=0.5)] <- 1
confusionMatrix(factor(GAM_To_3_full_fld4_1.2_pred), testing_To_3_full_data1.2$Response) # 0.7765
summary(GAM_To_3_full_fld2_1.2)


anova(GAM_To_3_full_fld4_1.1,GAM_To_3_full_fld4_1.2,test="Chisq")
# prefer final1