# manual k-fold and cross-validation
library(rpart) 

iris_binary <- subset(iris, Species != "setosa") # exclude virginica to make it truly binary classification


# Number of folds 
k <- 5 

# Shuffle data 
set.seed(123) 
indices <- sample(1:nrow(iris_binary)) 
folds <- cut(seq(1, nrow(iris_binary)), breaks = k, labels = FALSE) 

# Store accuracy for each fold 
accuracy_vector <- numeric(k) 

for (i in 1:k) { 
  # Define validation and training sets 
  val_indices <- which(folds == i) 
  train_indices <- setdiff(1:nrow(iris_binary), val_indices) 
  
  train_data <- iris_binary[train_indices, ] 
  val_data <- iris_binary[val_indices, ] 
  
  # Train model 
  model <- rpart(Species ~ ., data = train_data, method = "class") 
  
  # Predict on validation set 
  preds <- predict(model, val_data, type = "class") 
  
  # Calculate accuracy 
  acc <- mean(preds == val_data$Species) 
  accuracy_vector[i] <- acc 
  cat(sprintf("Fold %d accuracy: %.3f\n", i, acc)) 
} 

# Average accuracy across all folds and print it
mean_accuracy <- mean(accuracy_vector) 




##############################################################
##############################################################
##############################################################


# Load necessary library 
library(caret) 

# Set seed for reproducibility 
set.seed(123) 
# Load and prepare the data 
data(iris) 
iris_binary <- subset(iris, Species != "setosa") # exclude virginica to make it truly binary classification
# Convert Species to factor with two levels 
iris_binary$Species <- factor(iris_binary$Species) 

# Define training control with 5-fold cross-validation 
train_control <- trainControl(method = "cv", number = 5) 

# Train the model with cross-validation 
cv_model <- train(Species ~ ., data = iris_binary, 
                  method = "rpart", 
                  trControl = train_control) 

# Display results 
print(cv_model) 




# Repeated Cross-Validation Example 

# Define control with 10 folds repeated 3 times 
repeated_cv_control <- trainControl(method = "repeatedcv", number = 10, repeats = 3) 

# Train model with repeated cv 
repeated_cv_model <- train(Species ~ ., data = iris_binary, 
                           method = "rpart", 
                           trControl = repeated_cv_control) 

# Show results 
print(repeated_cv_model) 




# Leave-One-Out Cross-Validation (LOOCV)

# LOOCV setup 
loocv_control <- trainControl(method = "LOOCV") 

# Train with LOOCV 
loocv_model <- train(Species ~ ., data = iris_binary, 
                     method = "rpart", 
                     trControl = loocv_control) 

# Show results 
print(loocv_model) 


# how to find all the available training methods
names(getModelInfo())