########################################################################################
# 1) Create MC_data
########################################################################################
set.seed(123)
mydata = Model_data_1.1
MC_data1 <- data.frame(cbind(mydata$FID,mydata$X, mydata$Y, mydata$lu2010, mydata$lu2015))
colnames(MC_data1) <- c("ID", "X", "Y","lu2010","lu2015")
Nine_to_9 = filter(mydata, lu2010==9 & lu2015==9)
# reduce the size of the dataset
MC_data2 = rbind(To_1[,c(4,5)],To_2_full[,c(4,5)],To_3_full[,c(4,5)],To_4_full[,c(4,5)],
                 To_5[,c(4,5)],To_6[,c(4,5)],To_7_full[,c(4,5)],To_8[,c(4,5)],
                 Nine_to_9[sample(nrow(Nine_to_9),500,replace=FALSE),c(4,5)],
                 To_10[,c(4,5)],To_11_full[,c(4,5)])
MC_data2$ID = c(1:nrow(MC_data2))
# write.csv(MC_data2, "MC_data(full).csv")

########################################################################################
# 2) Contruct traning set with 70% of data being randomly selected
########################################################################################
library(caret)
set.seed(123)

Train_MC = sample(MC_data_full_$ID, floor(nrow(MC_data_full_)*0.7), replace=FALSE)
training_MC <- MC_data_full_[Train_MC, ]
dim(training_MC) 
testing_MC <- MC_data_full_[-Train_MC, ]
dim(testing_MC)

# make lu2010 and lu2015 as categorical variables (i.e., factors)
training_State1 <- as.factor(training_MC$lu2010)
training_State2 <- as.factor(training_MC$lu2015)
testing_State1 <- as.factor(testing_MC$lu2010)
testing_State2 <- as.factor(testing_MC$lu2015)

training_MC$lu2010 <- training_State1
training_MC$lu2015 <- training_State2
testing_MC$lu2010 <- testing_State1
testing_MC$lu2015 <- testing_State2

# check
str(training_MC); str(testing_MC)

########################################################################################
# 3) Create 10-fold of subsample datasets
########################################################################################
tic("model fitting")
MC_train <- training_MC
library(caret)
# create index of elements in each fold
set.seed((123))
flds <- createFolds(MC_train$ID, k = 10, list = TRUE, returnTrain = FALSE)

########################################################################################
# 4) Create training set and test set for each fold of cross validation
########################################################################################
test_sub1 <- MC_train[flds$Fold01,]
length(test_sub1$ID) 
train_sub1 <- MC_train[-flds$Fold01,] 

test_sub2 <- MC_train[flds$Fold02,]
length(test_sub2$ID) 
train_sub2 <- MC_train[-flds$Fold02,] 

test_sub3 <- MC_train[flds$Fold03,]
length(test_sub3$ID) 
train_sub3 <- MC_train[-flds$Fold03,]

test_sub4 <- MC_train[flds$Fold04,]
length(test_sub4$ID) 
train_sub4 <- MC_train[-flds$Fold04,]

test_sub5 <- MC_train[flds$Fold05,]
length(test_sub5$ID) 
train_sub5 <- MC_train[-flds$Fold05,]

test_sub6 <- MC_train[flds$Fold06,]
length(test_sub6$ID) 
train_sub6 <- MC_train[-flds$Fold06,]

test_sub7 <- MC_train[flds$Fold07,]
length(test_sub7$ID) 
train_sub7 <- MC_train[-flds$Fold07,]

test_sub8 <- MC_train[flds$Fold08,]
length(test_sub8$ID) 
train_sub8 <- MC_train[-flds$Fold08,]

test_sub9 <- MC_train[flds$Fold09,]
length(test_sub9$ID) 
train_sub9 <- MC_train[-flds$Fold09,]

test_sub10 <- MC_train[flds$Fold10,]
length(test_sub10$ID) 
train_sub10 <- MC_train[-flds$Fold10,]

########################################################################################
# 5) Create ten sets of random data which has the range of >=0 and <=1.
#    These will be used to determine the land use type at each location.
########################################################################################
set.seed(123)
rand_sub1 <- runif(length(test_sub1$ID), min=0, max=1)
rand_sub2 <- runif(length(test_sub2$ID), min=0, max=1)
rand_sub3 <- runif(length(test_sub3$ID), min=0, max=1)
rand_sub4 <- runif(length(test_sub4$ID), min=0, max=1)
rand_sub5 <- runif(length(test_sub5$ID), min=0, max=1)
rand_sub6 <- runif(length(test_sub6$ID), min=0, max=1)
rand_sub7 <- runif(length(test_sub7$ID), min=0, max=1)
rand_sub8 <- runif(length(test_sub8$ID), min=0, max=1)
rand_sub9 <- runif(length(test_sub9$ID), min=0, max=1)
rand_sub10 <- runif(length(test_sub10$ID), min=0, max=1)

########################################################################################
# 6) Create contingency tables for each fold of data
########################################################################################
tbl_sub1 <- table(train_sub1$lu2010, train_sub1$lu2015)
tbl_sub2 <- table(train_sub2$lu2010, train_sub2$lu2015)
tbl_sub3 <- table(train_sub3$lu2010, train_sub3$lu2015)
tbl_sub4 <- table(train_sub4$lu2010, train_sub4$lu2015)
tbl_sub5 <- table(train_sub5$lu2010, train_sub5$lu2015)
tbl_sub6 <- table(train_sub6$lu2010, train_sub6$lu2015)
tbl_sub7 <- table(train_sub7$lu2010, train_sub7$lu2015)
tbl_sub8 <- table(train_sub8$lu2010, train_sub8$lu2015)
tbl_sub9 <- table(train_sub9$lu2010, train_sub9$lu2015)
tbl_sub10 <- table(train_sub10$lu2010, train_sub10$lu2015)

########################################################################################
# 7) Construct transition probability matrix using contigency tables
########################################################################################
freq_sub1 <- round(prop.table(tbl_sub1, 1),4)
freq_sub2 <- round(prop.table(tbl_sub2, 1),4)
freq_sub3 <- round(prop.table(tbl_sub3, 1),4)
freq_sub4 <- round(prop.table(tbl_sub4, 1),4)
freq_sub5 <- round(prop.table(tbl_sub5, 1),4)
freq_sub6 <- round(prop.table(tbl_sub6, 1),4)
freq_sub7 <- round(prop.table(tbl_sub7, 1),4)
freq_sub8 <- round(prop.table(tbl_sub8, 1),4)
freq_sub9 <- round(prop.table(tbl_sub9, 1),4)
freq_sub10 <- round(prop.table(tbl_sub10, 1),4)

########################################################################################
# 8) Roulette Wheel Selection Approach
########################################################################################
# subset 1
range_sub1=matrix(0, nrow=11,ncol=11)
for (i in 1:11){
  for (j in 1:11){
    range_sub1[i,j]=sum(freq_sub1[i,1:j])}
}

clas_sub1=NULL
for (i in 1:length(rand_sub1)){
  for (j in 1:11){
    if (test_sub1$lu2010[i]==j) {clas_sub1[i]=which(rand_sub1[i]<=range_sub1[j,])[1]}
  }
}

comp_sub1 <- ifelse(test_sub1$lu2015==clas_sub1, "Y","N")
prop.table(table(comp_sub1))
Acc_sub1 <- prop.table(table(comp_sub1))[2]

##############################################################
# subset 2
range_sub2=matrix(0, nrow=11,ncol=11)
for (i in 1:11){
  for (j in 1:11){
    range_sub2[i,j]=sum(freq_sub2[i,1:j])}
}

clas_sub2=NULL
for (i in 1:length(rand_sub2)){
  for (j in 1:11){
    if (test_sub2$lu2010[i]==j) {clas_sub2[i]=which(rand_sub2[i]<=range_sub2[j,])[1]}
  }
}

comp_sub2 <- ifelse(test_sub2$lu2015==clas_sub2, "Y","N")
prop.table(table(comp_sub2))
Acc_sub2 <- prop.table(table(comp_sub2))[2]

##############################################################
# subset 3
range_sub3=matrix(0, nrow=11,ncol=11)
for (i in 1:11){
  for (j in 1:11){
    range_sub3[i,j]=sum(freq_sub3[i,1:j])}
}

clas_sub3=NULL
for (i in 1:length(rand_sub3)){
  for (j in 1:11){
    if (test_sub3$lu2010[i]==j) {clas_sub3[i]=which(rand_sub3[i]<=range_sub3[j,])[1]}
  }
}

comp_sub3 <- ifelse(test_sub3$lu2015==clas_sub3, "Y","N")
prop.table(table(comp_sub3))
Acc_sub3 <- prop.table(table(comp_sub3))[2]

##############################################################
# subset 4
range_sub4=matrix(0, nrow=11,ncol=11)
for (i in 1:11){
  for (j in 1:11){
    range_sub4[i,j]=sum(freq_sub4[i,1:j])}
}

clas_sub4=NULL
for (i in 1:length(rand_sub4)){
  for (j in 1:11){
    if (test_sub4$lu2010[i]==j) {clas_sub4[i]=which(rand_sub4[i]<=range_sub4[j,])[1]}
  }
}

comp_sub4 <- ifelse(test_sub4$lu2015==clas_sub4, "Y","N")
prop.table(table(comp_sub4))
Acc_sub4 <- prop.table(table(comp_sub4))[2]

##############################################################
# subset 5
range_sub5=matrix(0, nrow=11,ncol=11)
for (i in 1:11){
  for (j in 1:11){
    range_sub5[i,j]=sum(freq_sub5[i,1:j])}
}

clas_sub5=NULL
for (i in 1:length(rand_sub5)){
  for (j in 1:11){
    if (test_sub5$lu2010[i]==j) {clas_sub5[i]=which(rand_sub5[i]<=range_sub5[j,])[1]}
  }
}

comp_sub5 <- ifelse(test_sub5$lu2015==clas_sub5, "Y","N")
prop.table(table(comp_sub5))
Acc_sub5 <- prop.table(table(comp_sub5))[2]

##############################################################
# subset 6
range_sub6=matrix(0, nrow=11,ncol=11)
for (i in 1:11){
  for (j in 1:11){
    range_sub6[i,j]=sum(freq_sub6[i,1:j])}
}

clas_sub6=NULL
for (i in 1:length(rand_sub6)){
  for (j in 1:11){
    if (test_sub6$lu2010[i]==j) {clas_sub6[i]=which(rand_sub6[i]<=range_sub6[j,])[1]}
  }
}

comp_sub6 <- ifelse(test_sub6$lu2015==clas_sub6, "Y","N")
prop.table(table(comp_sub6))
Acc_sub6 <- prop.table(table(comp_sub6))[2]

##############################################################
# subset 7
range_sub7=matrix(0, nrow=11,ncol=11)
for (i in 1:11){
  for (j in 1:11){
    range_sub7[i,j]=sum(freq_sub7[i,1:j])}
}

clas_sub7=NULL
for (i in 1:length(rand_sub7)){
  for (j in 1:11){
    if (test_sub7$lu2010[i]==j) {clas_sub7[i]=which(rand_sub7[i]<=range_sub7[j,])[1]}
  }
}

comp_sub7 <- ifelse(test_sub7$lu2015==clas_sub7, "Y","N")
prop.table(table(comp_sub7))
Acc_sub7 <- prop.table(table(comp_sub7))[2]

##############################################################
# subset 8
range_sub8=matrix(0, nrow=11,ncol=11)
for (i in 1:11){
  for (j in 1:11){
    range_sub8[i,j]=sum(freq_sub8[i,1:j])}
}

clas_sub8=NULL
for (i in 1:length(rand_sub8)){
  for (j in 1:11){
    if (test_sub8$lu2010[i]==j) {clas_sub8[i]=which(rand_sub8[i]<=range_sub8[j,])[1]}
  }
}

comp_sub8 <- ifelse(test_sub8$lu2015==clas_sub8, "Y","N")
prop.table(table(comp_sub8))
Acc_sub8 <- prop.table(table(comp_sub8))[2]

##############################################################
# subset 9
range_sub9=matrix(0, nrow=11,ncol=11)
for (i in 1:11){
  for (j in 1:11){
    range_sub9[i,j]=sum(freq_sub9[i,1:j])}
}

clas_sub9=NULL
for (i in 1:length(rand_sub9)){
  for (j in 1:11){
    if (test_sub9$lu2010[i]==j) {clas_sub9[i]=which(rand_sub9[i]<=range_sub9[j,])[1]}
  }
}

comp_sub9 <- ifelse(test_sub9$lu2015==clas_sub9, "Y","N")
prop.table(table(comp_sub9))
Acc_sub9 <- prop.table(table(comp_sub9))[2]

##############################################################
# subset 10
range_sub10=matrix(0, nrow=11,ncol=11)
for (i in 1:11){
  for (j in 1:11){
    range_sub10[i,j]=sum(freq_sub10[i,1:j])}
}

clas_sub10=NULL
for (i in 1:length(rand_sub10)){
  for (j in 1:11){
    if (test_sub10$lu2010[i]==j) {clas_sub10[i]=which(rand_sub10[i]<=range_sub10[j,])[1]}
  }
}

comp_sub10 <- ifelse(test_sub10$lu2015==clas_sub10, "Y","N")
prop.table(table(comp_sub10))
Acc_sub10 <- prop.table(table(comp_sub10))[2]

########################################################################################
# 9) Overall Average accuracy of 10-fold subsamples
########################################################################################
Acc_sub_avg <- sum(Acc_sub1,Acc_sub2,Acc_sub3,Acc_sub4,Acc_sub5,
                   Acc_sub6,Acc_sub7,Acc_sub8,Acc_sub9,Acc_sub10)/10

########################################################################################
# 10) Averaged accuracy for 10-fold subsamples by 2015 land-use types
########################################################################################
# LU 1
LU1_sub1 = prop.table(table(ifelse(test_sub1[which(test_sub1$lu2015==1),]$lu2015 == clas_sub1[which(test_sub1$lu2015==1)], "Y", "N")))[2] 
LU1_sub2 = prop.table(table(ifelse(test_sub2[which(test_sub2$lu2015==1),]$lu2015 == clas_sub2[which(test_sub2$lu2015==1)], "Y", "N")))[2] 
LU1_sub3 = prop.table(table(ifelse(test_sub3[which(test_sub3$lu2015==1),]$lu2015 == clas_sub3[which(test_sub3$lu2015==1)], "Y", "N")))[2] 
LU1_sub4 = prop.table(table(ifelse(test_sub4[which(test_sub4$lu2015==1),]$lu2015 == clas_sub4[which(test_sub4$lu2015==1)], "Y", "N")))[2] 
LU1_sub5 = prop.table(table(ifelse(test_sub5[which(test_sub5$lu2015==1),]$lu2015 == clas_sub5[which(test_sub5$lu2015==1)], "Y", "N")))[2] 
LU1_sub6 = prop.table(table(ifelse(test_sub6[which(test_sub6$lu2015==1),]$lu2015 == clas_sub6[which(test_sub6$lu2015==1)], "Y", "N")))[2] 
LU1_sub7 = prop.table(table(ifelse(test_sub7[which(test_sub7$lu2015==1),]$lu2015 == clas_sub7[which(test_sub7$lu2015==1)], "Y", "N")))[2] 
LU1_sub8 = prop.table(table(ifelse(test_sub8[which(test_sub8$lu2015==1),]$lu2015 == clas_sub8[which(test_sub8$lu2015==1)], "Y", "N")))[2] 
LU1_sub9 = prop.table(table(ifelse(test_sub9[which(test_sub9$lu2015==1),]$lu2015 == clas_sub9[which(test_sub9$lu2015==1)], "Y", "N")))[2] 
LU1_sub10 = prop.table(table(ifelse(test_sub10[which(test_sub10$lu2015==1),]$lu2015 == clas_sub10[which(test_sub10$lu2015==1)], "Y", "N")))[2] 
LU1_average = mean(c(LU1_sub1,LU1_sub2,LU1_sub3,LU1_sub4,LU1_sub5,LU1_sub6,LU1_sub7,LU1_sub8,LU1_sub9,LU1_sub10))

# LU 2
LU2_sub1 = prop.table(table(ifelse(test_sub1[which(test_sub1$lu2015==2),]$lu2015 == clas_sub1[which(test_sub1$lu2015==2)], "Y", "N")))[2] 
LU2_sub2 = prop.table(table(ifelse(test_sub2[which(test_sub2$lu2015==2),]$lu2015 == clas_sub2[which(test_sub2$lu2015==2)], "Y", "N")))[2] 
LU2_sub3 = prop.table(table(ifelse(test_sub3[which(test_sub3$lu2015==2),]$lu2015 == clas_sub3[which(test_sub3$lu2015==2)], "Y", "N")))[2] 
LU2_sub4 = prop.table(table(ifelse(test_sub4[which(test_sub4$lu2015==2),]$lu2015 == clas_sub4[which(test_sub4$lu2015==2)], "Y", "N")))[2] 
LU2_sub5 = prop.table(table(ifelse(test_sub5[which(test_sub5$lu2015==2),]$lu2015 == clas_sub5[which(test_sub5$lu2015==2)], "Y", "N")))[2] 
LU2_sub6 = prop.table(table(ifelse(test_sub6[which(test_sub6$lu2015==2),]$lu2015 == clas_sub6[which(test_sub6$lu2015==2)], "Y", "N")))[2]
LU2_sub7 = prop.table(table(ifelse(test_sub7[which(test_sub7$lu2015==2),]$lu2015 == clas_sub7[which(test_sub7$lu2015==2)], "Y", "N")))[2] 
LU2_sub8 = prop.table(table(ifelse(test_sub8[which(test_sub8$lu2015==2),]$lu2015 == clas_sub8[which(test_sub8$lu2015==2)], "Y", "N")))[2] 
LU2_sub9 = prop.table(table(ifelse(test_sub9[which(test_sub9$lu2015==2),]$lu2015 == clas_sub9[which(test_sub9$lu2015==2)], "Y", "N")))[2] 
LU2_sub10 = prop.table(table(ifelse(test_sub10[which(test_sub10$lu2015==2),]$lu2015 == clas_sub10[which(test_sub10$lu2015==2)], "Y", "N")))[2] 
LU2_average = mean(c(LU2_sub1,LU2_sub2,LU2_sub3,LU2_sub4,LU2_sub5,LU2_sub6,LU2_sub7,LU2_sub8,LU2_sub9,LU2_sub10))

# LU 3
LU3_sub1 = prop.table(table(ifelse(test_sub1[which(test_sub1$lu2015==3),]$lu2015 == clas_sub1[which(test_sub1$lu2015==3)], "Y", "N")))[2] 
LU3_sub2 = prop.table(table(ifelse(test_sub2[which(test_sub2$lu2015==3),]$lu2015 == clas_sub2[which(test_sub2$lu2015==3)], "Y", "N")))[2] 
LU3_sub3 = prop.table(table(ifelse(test_sub3[which(test_sub3$lu2015==3),]$lu2015 == clas_sub3[which(test_sub3$lu2015==3)], "Y", "N")))[2] 
LU3_sub4 = prop.table(table(ifelse(test_sub4[which(test_sub4$lu2015==3),]$lu2015 == clas_sub4[which(test_sub4$lu2015==3)], "Y", "N")))[2]
LU3_sub5 = prop.table(table(ifelse(test_sub5[which(test_sub5$lu2015==3),]$lu2015 == clas_sub5[which(test_sub5$lu2015==3)], "Y", "N")))[2] 
LU3_sub6 = prop.table(table(ifelse(test_sub6[which(test_sub6$lu2015==3),]$lu2015 == clas_sub6[which(test_sub6$lu2015==3)], "Y", "N")))[2] 
LU3_sub7 = prop.table(table(ifelse(test_sub7[which(test_sub7$lu2015==3),]$lu2015 == clas_sub7[which(test_sub7$lu2015==3)], "Y", "N")))[2] 
LU3_sub8 = prop.table(table(ifelse(test_sub8[which(test_sub8$lu2015==3),]$lu2015 == clas_sub8[which(test_sub8$lu2015==3)], "Y", "N")))[2] 
LU3_sub9 = prop.table(table(ifelse(test_sub9[which(test_sub9$lu2015==3),]$lu2015 == clas_sub9[which(test_sub9$lu2015==3)], "Y", "N")))[2] 
LU3_sub10 = prop.table(table(ifelse(test_sub10[which(test_sub10$lu2015==3),]$lu2015 == clas_sub10[which(test_sub10$lu2015==3)], "Y", "N")))[2] 
LU3_average = mean(c(LU3_sub1, LU3_sub2,LU3_sub3,LU3_sub4,LU3_sub5,LU3_sub6,LU3_sub7,LU3_sub8,LU3_sub9,LU3_sub10))

# LU 4
LU4_sub1 = prop.table(table(ifelse(test_sub1[which(test_sub1$lu2015==4),]$lu2015 == clas_sub1[which(test_sub1$lu2015==4)], "Y", "N")))[2] 
LU4_sub2 = prop.table(table(ifelse(test_sub2[which(test_sub2$lu2015==4),]$lu2015 == clas_sub2[which(test_sub2$lu2015==4)], "Y", "N")))[2] 
LU4_sub3 = prop.table(table(ifelse(test_sub3[which(test_sub3$lu2015==4),]$lu2015 == clas_sub3[which(test_sub3$lu2015==4)], "Y", "N")))[2] 
LU4_sub4 = prop.table(table(ifelse(test_sub4[which(test_sub4$lu2015==4),]$lu2015 == clas_sub4[which(test_sub4$lu2015==4)], "Y", "N")))[2] 
LU4_sub5 = prop.table(table(ifelse(test_sub5[which(test_sub5$lu2015==4),]$lu2015 == clas_sub5[which(test_sub5$lu2015==4)], "Y", "N")))[2] 
LU4_sub6 = prop.table(table(ifelse(test_sub6[which(test_sub6$lu2015==4),]$lu2015 == clas_sub6[which(test_sub6$lu2015==4)], "Y", "N")))[2] 
LU4_sub7 = prop.table(table(ifelse(test_sub7[which(test_sub7$lu2015==4),]$lu2015 == clas_sub7[which(test_sub7$lu2015==4)], "Y", "N")))[2] 
LU4_sub8 = prop.table(table(ifelse(test_sub8[which(test_sub8$lu2015==4),]$lu2015 == clas_sub8[which(test_sub8$lu2015==4)], "Y", "N")))[2] 
LU4_sub9 = prop.table(table(ifelse(test_sub9[which(test_sub9$lu2015==4),]$lu2015 == clas_sub9[which(test_sub9$lu2015==4)], "Y", "N")))[2] 
LU4_sub10 = prop.table(table(ifelse(test_sub10[which(test_sub10$lu2015==4),]$lu2015 == clas_sub10[which(test_sub10$lu2015==4)], "Y", "N")))[2] 
LU4_average = mean(c(LU4_sub1,LU4_sub2,LU4_sub3,LU4_sub4,LU4_sub5,LU4_sub6,LU4_sub7,LU4_sub8,LU4_sub9,LU4_sub10))

# LU 5
LU5_sub1 = prop.table(table(ifelse(test_sub1[which(test_sub1$lu2015==5),]$lu2015 == clas_sub1[which(test_sub1$lu2015==5)], "Y", "N")))[2] 
LU5_sub2 = prop.table(table(ifelse(test_sub2[which(test_sub2$lu2015==5),]$lu2015 == clas_sub2[which(test_sub2$lu2015==5)], "Y", "N")))[2] 
LU5_sub3 = prop.table(table(ifelse(test_sub3[which(test_sub3$lu2015==5),]$lu2015 == clas_sub3[which(test_sub3$lu2015==5)], "Y", "N")))[2] 
LU5_sub4 = prop.table(table(ifelse(test_sub4[which(test_sub4$lu2015==5),]$lu2015 == clas_sub4[which(test_sub4$lu2015==5)], "Y", "N")))[2] 
LU5_sub5 = prop.table(table(ifelse(test_sub5[which(test_sub5$lu2015==5),]$lu2015 == clas_sub5[which(test_sub5$lu2015==5)], "Y", "N")))[2]
LU5_sub6 = prop.table(table(ifelse(test_sub6[which(test_sub6$lu2015==5),]$lu2015 == clas_sub6[which(test_sub6$lu2015==5)], "Y", "N")))[2] 
LU5_sub7 = prop.table(table(ifelse(test_sub7[which(test_sub7$lu2015==5),]$lu2015 == clas_sub7[which(test_sub7$lu2015==5)], "Y", "N")))[2] 
LU5_sub8 = prop.table(table(ifelse(test_sub8[which(test_sub8$lu2015==5),]$lu2015 == clas_sub8[which(test_sub8$lu2015==5)], "Y", "N")))[2] 
LU5_sub9 = prop.table(table(ifelse(test_sub9[which(test_sub9$lu2015==5),]$lu2015 == clas_sub9[which(test_sub9$lu2015==5)], "Y", "N")))[2] 
LU5_sub10 = prop.table(table(ifelse(test_sub10[which(test_sub10$lu2015==5),]$lu2015 == clas_sub10[which(test_sub10$lu2015==5)], "Y", "N")))[2] 
LU5_average = mean(c(LU5_sub1,LU5_sub2,LU5_sub3,LU5_sub4,LU5_sub5,LU5_sub6,LU5_sub7,LU5_sub8,LU5_sub9,LU5_sub10)) 

# LU 6
LU6_sub1 = prop.table(table(ifelse(test_sub1[which(test_sub1$lu2015==6),]$lu2015 == clas_sub1[which(test_sub1$lu2015==6)], "Y", "N")))[2] # 1
LU6_sub2 = prop.table(table(ifelse(test_sub2[which(test_sub2$lu2015==6),]$lu2015 == clas_sub2[which(test_sub2$lu2015==6)], "Y", "N")))[2] # 1
LU6_sub3 = prop.table(table(ifelse(test_sub3[which(test_sub3$lu2015==6),]$lu2015 == clas_sub3[which(test_sub3$lu2015==6)], "Y", "N")))[2] # 1
LU6_sub4 = prop.table(table(ifelse(test_sub4[which(test_sub4$lu2015==6),]$lu2015 == clas_sub4[which(test_sub4$lu2015==6)], "Y", "N")))[2] # 0
LU6_sub5 = prop.table(table(ifelse(test_sub5[which(test_sub5$lu2015==6),]$lu2015 == clas_sub5[which(test_sub5$lu2015==6)], "Y", "N")))[2] # 0
LU6_sub6 = prop.table(table(ifelse(test_sub6[which(test_sub6$lu2015==6),]$lu2015 == clas_sub6[which(test_sub6$lu2015==6)], "Y", "N")))[2] # 0
LU6_sub7 = prop.table(table(ifelse(test_sub7[which(test_sub7$lu2015==6),]$lu2015 == clas_sub7[which(test_sub7$lu2015==6)], "Y", "N")))[2] # 1 
LU6_sub8 = prop.table(table(ifelse(test_sub8[which(test_sub8$lu2015==6),]$lu2015 == clas_sub8[which(test_sub8$lu2015==6)], "Y", "N")))[2] # 1
LU6_sub9 = prop.table(table(ifelse(test_sub9[which(test_sub9$lu2015==6),]$lu2015 == clas_sub9[which(test_sub9$lu2015==6)], "Y", "N")))[2] # 0
LU6_sub10 = prop.table(table(ifelse(test_sub10[which(test_sub10$lu2015==6),]$lu2015 == clas_sub10[which(test_sub10$lu2015==6)], "Y", "N")))[2] # 1
LU6_average = 6/10

# LU 7
LU7_sub1 = prop.table(table(ifelse(test_sub1[which(test_sub1$lu2015==7),]$lu2015 == clas_sub1[which(test_sub1$lu2015==7)], "Y", "N")))[2] 
LU7_sub2 = prop.table(table(ifelse(test_sub2[which(test_sub2$lu2015==7),]$lu2015 == clas_sub2[which(test_sub2$lu2015==7)], "Y", "N")))[2] 
LU7_sub3 = prop.table(table(ifelse(test_sub3[which(test_sub3$lu2015==7),]$lu2015 == clas_sub3[which(test_sub3$lu2015==7)], "Y", "N")))[2] 
LU7_sub4 = prop.table(table(ifelse(test_sub4[which(test_sub4$lu2015==7),]$lu2015 == clas_sub4[which(test_sub4$lu2015==7)], "Y", "N")))[2] 
LU7_sub5 = prop.table(table(ifelse(test_sub5[which(test_sub5$lu2015==7),]$lu2015 == clas_sub5[which(test_sub5$lu2015==7)], "Y", "N")))[2] 
LU7_sub6 = prop.table(table(ifelse(test_sub6[which(test_sub6$lu2015==7),]$lu2015 == clas_sub6[which(test_sub6$lu2015==7)], "Y", "N")))[2] 
LU7_sub7 = prop.table(table(ifelse(test_sub7[which(test_sub7$lu2015==7),]$lu2015 == clas_sub7[which(test_sub7$lu2015==7)], "Y", "N")))[2]
LU7_sub8 = prop.table(table(ifelse(test_sub8[which(test_sub8$lu2015==7),]$lu2015 == clas_sub8[which(test_sub8$lu2015==7)], "Y", "N")))[2] 
LU7_sub9 = prop.table(table(ifelse(test_sub9[which(test_sub9$lu2015==7),]$lu2015 == clas_sub9[which(test_sub9$lu2015==7)], "Y", "N")))[2] 
LU7_sub10 = prop.table(table(ifelse(test_sub10[which(test_sub10$lu2015==7),]$lu2015 == clas_sub10[which(test_sub10$lu2015==7)], "Y", "N")))[2] 
LU7_average = mean(c(LU7_sub1,LU7_sub2,LU7_sub3,LU7_sub4,LU7_sub5,LU7_sub6,LU7_sub7,LU7_sub8,LU7_sub9,LU7_sub10))

# LU 8
LU8_sub1 = prop.table(table(ifelse(test_sub1[which(test_sub1$lu2015==8),]$lu2015 == clas_sub1[which(test_sub1$lu2015==8)], "Y", "N")))[2]
LU8_sub2 = prop.table(table(ifelse(test_sub2[which(test_sub2$lu2015==8),]$lu2015 == clas_sub2[which(test_sub2$lu2015==8)], "Y", "N")))[2] 
LU8_sub3 = prop.table(table(ifelse(test_sub3[which(test_sub3$lu2015==8),]$lu2015 == clas_sub3[which(test_sub3$lu2015==8)], "Y", "N")))[2]
LU8_sub4 = prop.table(table(ifelse(test_sub4[which(test_sub4$lu2015==8),]$lu2015 == clas_sub4[which(test_sub4$lu2015==8)], "Y", "N")))[2] 
LU8_sub5 = prop.table(table(ifelse(test_sub5[which(test_sub5$lu2015==8),]$lu2015 == clas_sub5[which(test_sub5$lu2015==8)], "Y", "N")))[2] 
LU8_sub6 = prop.table(table(ifelse(test_sub6[which(test_sub6$lu2015==8),]$lu2015 == clas_sub6[which(test_sub6$lu2015==8)], "Y", "N")))[2] 
LU8_sub7 = prop.table(table(ifelse(test_sub7[which(test_sub7$lu2015==8),]$lu2015 == clas_sub7[which(test_sub7$lu2015==8)], "Y", "N")))[2]
LU8_sub8 = prop.table(table(ifelse(test_sub8[which(test_sub8$lu2015==8),]$lu2015 == clas_sub8[which(test_sub8$lu2015==8)], "Y", "N")))[2] 
LU8_sub9 = prop.table(table(ifelse(test_sub9[which(test_sub9$lu2015==8),]$lu2015 == clas_sub9[which(test_sub9$lu2015==8)], "Y", "N")))[2] 
LU8_sub10 = prop.table(table(ifelse(test_sub10[which(test_sub10$lu2015==8),]$lu2015 == clas_sub10[which(test_sub10$lu2015==8)], "Y", "N")))[2] 
LU8_average = mean(c(LU8_sub1,LU8_sub2,LU8_sub3,LU8_sub4,LU8_sub5,LU8_sub6,LU8_sub7,LU8_sub8,LU8_sub9,LU8_sub10))

# LU 9
LU9_sub1 = prop.table(table(ifelse(test_sub1[which(test_sub1$lu2015==9),]$lu2015 == clas_sub1[which(test_sub1$lu2015==9)], "Y", "N")))[2] 
LU9_sub2 = prop.table(table(ifelse(test_sub2[which(test_sub2$lu2015==9),]$lu2015 == clas_sub2[which(test_sub2$lu2015==9)], "Y", "N")))[2] 
LU9_sub3 = prop.table(table(ifelse(test_sub3[which(test_sub3$lu2015==9),]$lu2015 == clas_sub3[which(test_sub3$lu2015==9)], "Y", "N")))[2] 
LU9_sub4 = prop.table(table(ifelse(test_sub4[which(test_sub4$lu2015==9),]$lu2015 == clas_sub4[which(test_sub4$lu2015==9)], "Y", "N")))[2] 
LU9_sub5 = prop.table(table(ifelse(test_sub5[which(test_sub5$lu2015==9),]$lu2015 == clas_sub5[which(test_sub5$lu2015==9)], "Y", "N")))[2] 
LU9_sub6 = prop.table(table(ifelse(test_sub6[which(test_sub6$lu2015==9),]$lu2015 == clas_sub6[which(test_sub6$lu2015==9)], "Y", "N")))[2] 
LU9_sub7 = prop.table(table(ifelse(test_sub7[which(test_sub7$lu2015==9),]$lu2015 == clas_sub7[which(test_sub7$lu2015==9)], "Y", "N")))[2] # 1
LU9_sub8 = prop.table(table(ifelse(test_sub8[which(test_sub8$lu2015==9),]$lu2015 == clas_sub8[which(test_sub8$lu2015==9)], "Y", "N")))[2] 
LU9_sub9 = prop.table(table(ifelse(test_sub9[which(test_sub9$lu2015==9),]$lu2015 == clas_sub9[which(test_sub9$lu2015==9)], "Y", "N")))[2] 
LU9_sub10 = prop.table(table(ifelse(test_sub10[which(test_sub10$lu2015==9),]$lu2015 == clas_sub10[which(test_sub10$lu2015==9)], "Y", "N")))[2] 
LU9_average = mean(c(LU9_sub1,LU9_sub2,LU9_sub3,LU9_sub4,LU9_sub5,LU9_sub6,1,LU9_sub8,LU9_sub9,LU9_sub10))

# LU 10
LU10_sub1 = prop.table(table(ifelse(test_sub1[which(test_sub1$lu2015==10),]$lu2015 == clas_sub1[which(test_sub1$lu2015==10)], "Y", "N")))[2] 
LU10_sub2 = prop.table(table(ifelse(test_sub2[which(test_sub2$lu2015==10),]$lu2015 == clas_sub2[which(test_sub2$lu2015==10)], "Y", "N")))[2] 
LU10_sub3 = prop.table(table(ifelse(test_sub3[which(test_sub3$lu2015==10),]$lu2015 == clas_sub3[which(test_sub3$lu2015==10)], "Y", "N")))[2] 
LU10_sub4 = prop.table(table(ifelse(test_sub4[which(test_sub4$lu2015==10),]$lu2015 == clas_sub4[which(test_sub4$lu2015==10)], "Y", "N")))[2] 
LU10_sub5 = prop.table(table(ifelse(test_sub5[which(test_sub5$lu2015==10),]$lu2015 == clas_sub5[which(test_sub5$lu2015==10)], "Y", "N")))[2] 
LU10_sub6 = prop.table(table(ifelse(test_sub6[which(test_sub6$lu2015==10),]$lu2015 == clas_sub6[which(test_sub6$lu2015==10)], "Y", "N")))[2] 
LU10_sub7 = prop.table(table(ifelse(test_sub7[which(test_sub7$lu2015==10),]$lu2015 == clas_sub7[which(test_sub7$lu2015==10)], "Y", "N")))[2] 
LU10_sub8 = prop.table(table(ifelse(test_sub8[which(test_sub8$lu2015==10),]$lu2015 == clas_sub8[which(test_sub8$lu2015==10)], "Y", "N")))[2] 
LU10_sub9 = prop.table(table(ifelse(test_sub9[which(test_sub9$lu2015==10),]$lu2015 == clas_sub9[which(test_sub9$lu2015==10)], "Y", "N")))[2] 
LU10_sub10 = prop.table(table(ifelse(test_sub10[which(test_sub10$lu2015==10),]$lu2015 == clas_sub10[which(test_sub10$lu2015==10)], "Y", "N")))[2] 
LU10_average = mean(c(LU10_sub1,LU10_sub2,LU10_sub3,0,LU10_sub5,LU10_sub6,LU10_sub7,1,LU10_sub9,LU10_sub10)) 

# LU 11
LU11_sub1 = prop.table(table(ifelse(test_sub1[which(test_sub1$lu2015==11),]$lu2015 == clas_sub1[which(test_sub1$lu2015==11)], "Y", "N")))[2] 
LU11_sub2 = prop.table(table(ifelse(test_sub2[which(test_sub2$lu2015==11),]$lu2015 == clas_sub2[which(test_sub2$lu2015==11)], "Y", "N")))[2]
LU11_sub3 = prop.table(table(ifelse(test_sub3[which(test_sub3$lu2015==11),]$lu2015 == clas_sub3[which(test_sub3$lu2015==11)], "Y", "N")))[2] 
LU11_sub4 = prop.table(table(ifelse(test_sub4[which(test_sub4$lu2015==11),]$lu2015 == clas_sub4[which(test_sub4$lu2015==11)], "Y", "N")))[2] 
LU11_sub5 = prop.table(table(ifelse(test_sub5[which(test_sub5$lu2015==11),]$lu2015 == clas_sub5[which(test_sub5$lu2015==11)], "Y", "N")))[2] 
LU11_sub6 = prop.table(table(ifelse(test_sub6[which(test_sub6$lu2015==11),]$lu2015 == clas_sub6[which(test_sub6$lu2015==11)], "Y", "N")))[2] 
LU11_sub7 = prop.table(table(ifelse(test_sub7[which(test_sub7$lu2015==11),]$lu2015 == clas_sub7[which(test_sub7$lu2015==11)], "Y", "N")))[2] 
LU11_sub8 = prop.table(table(ifelse(test_sub8[which(test_sub8$lu2015==11),]$lu2015 == clas_sub8[which(test_sub8$lu2015==11)], "Y", "N")))[2] 
LU11_sub9 = prop.table(table(ifelse(test_sub9[which(test_sub9$lu2015==11),]$lu2015 == clas_sub9[which(test_sub9$lu2015==11)], "Y", "N")))[2] 
LU11_sub10 = prop.table(table(ifelse(test_sub10[which(test_sub10$lu2015==11),]$lu2015 == clas_sub10[which(test_sub10$lu2015==11)], "Y", "N")))[2] 
LU11_average = mean(c(LU11_sub1,LU11_sub2,LU11_sub3,LU11_sub4,LU11_sub5,LU11_sub6,LU11_sub7,LU11_sub8,LU11_sub9,LU11_sub10))

# Overall
LU_average = mean(c(LU1_average,LU2_average,LU3_average,LU4_average,LU5_average,LU6_average,LU7_average,LU8_average,LU9_average,LU10_average,LU11_average)) 
# 0.4582428

toc() # 17.42

########################################################################################
# 11) Overall Accuracy of the separated test set "testing_MC"
########################################################################################
dim(testing_MC)
set.seed(123)
rand_testing <- runif(length(testing_MC$ID), min=0, max=1)

# average the ten frequency tables
freq_avg <- (freq_sub1+freq_sub2+freq_sub3+freq_sub4+freq_sub5+freq_sub6+freq_sub7+freq_sub8+freq_sub9+freq_sub10)/10

range_testing=matrix(0, nrow=11,ncol=11)
for (i in 1:11){
  for (j in 1:11){
    range_testing[i,j]=sum(freq_avg[i,1:j])}
}

clas_testing=NULL
for (i in 1:length(rand_testing)){
  for (j in 1:11){
    if (testing_MC$lu2010[i]==j) {clas_testing[i]=which(rand_testing[i]<=range_testing[j,])[1]}
  }
}

comp_testing <- ifelse(testing_MC$lu2015==clas_testing, "Y","N")
prop.table(table(comp_testing))
Acc_testing <- prop.table(table(comp_testing))[2] # 0.5116672

########################################################################################
# 12) Accuracy of the separated test set "testing_MC" by 2015 land-use type
########################################################################################
LU1_testing = prop.table(table(ifelse(testing_MC[which(testing_MC$lu2015==1),]$lu2015 == clas_testing[which(testing_MC$lu2015==1)], "Y", "N")))[2] 
LU2_testing = prop.table(table(ifelse(testing_MC[which(testing_MC$lu2015==2),]$lu2015 == clas_testing[which(testing_MC$lu2015==2)], "Y", "N")))[2] 
LU3_testing = prop.table(table(ifelse(testing_MC[which(testing_MC$lu2015==3),]$lu2015 == clas_testing[which(testing_MC$lu2015==3)], "Y", "N")))[2] 
LU4_testing = prop.table(table(ifelse(testing_MC[which(testing_MC$lu2015==4),]$lu2015 == clas_testing[which(testing_MC$lu2015==4)], "Y", "N")))[2] 
LU5_testing = prop.table(table(ifelse(testing_MC[which(testing_MC$lu2015==5),]$lu2015 == clas_testing[which(testing_MC$lu2015==5)], "Y", "N")))[2] 
LU6_testing = prop.table(table(ifelse(testing_MC[which(testing_MC$lu2015==6),]$lu2015 == clas_testing[which(testing_MC$lu2015==6)], "Y", "N")))[2] 
LU7_testing = prop.table(table(ifelse(testing_MC[which(testing_MC$lu2015==7),]$lu2015 == clas_testing[which(testing_MC$lu2015==7)], "Y", "N")))[2] 
LU8_testing = prop.table(table(ifelse(testing_MC[which(testing_MC$lu2015==8),]$lu2015 == clas_testing[which(testing_MC$lu2015==8)], "Y", "N")))[2] 
LU9_testing = prop.table(table(ifelse(testing_MC[which(testing_MC$lu2015==9),]$lu2015 == clas_testing[which(testing_MC$lu2015==9)], "Y", "N")))[2] 
LU10_testing = prop.table(table(ifelse(testing_MC[which(testing_MC$lu2015==10),]$lu2015 == clas_testing[which(testing_MC$lu2015==10)], "Y", "N")))[2] 
LU11_testing = prop.table(table(ifelse(testing_MC[which(testing_MC$lu2015==11),]$lu2015 == clas_testing[which(testing_MC$lu2015==11)], "Y", "N")))[2] 
