library(furrr)
library(purrr)

plan(multisession, workers=8)

## 5-Fold MSE vs. 5-Fold RMSE

### simulate 5-fold MSE

set.seed(123)

### Suppose Model B is superior to Model A (just by a little)

MSE_ModelA <- rnorm(5, mean=100, sd=20)

MSE_ModelB <- rnorm(5, mean=98, sd=25)

CV_MSE_A <- mean(MSE_ModelA)

CV_MSE_B <- mean(MSE_ModelB)

### what if we took the RMSE in each fold rather than MSE?

rMSE_ModelA <- sqrt(MSE_ModelA)
rMSE_ModelB <- sqrt(MSE_ModelB)

CV_MSE_A_ALT <- mean(rMSE_ModelA)^2  ## convert back to MSE
CV_MSE_B_ALT <- mean(rMSE_ModelB)^2  ## convert back to MSE

### print

CV_MSE_A - CV_MSE_A_ALT
CV_MSE_B - CV_MSE_B_ALT

### estimate bias

doOne <- function(seedVal){
  set.seed(seedVal)
  
  MSE_ModelA <- rnorm(5, mean=1000, sd=20)
  
  MSE_ModelB <- rnorm(5, mean=980, sd=25)
  
  CV_MSE_A <- mean(MSE_ModelA)
  
  CV_MSE_B <- mean(MSE_ModelB)
  
  ### what if we took the RMSE in each fold rather than MSE?
  
  rMSE_ModelA <- sqrt(MSE_ModelA)
  rMSE_ModelB <- sqrt(MSE_ModelB)
  
  CV_MSE_A_ALT <- mean(rMSE_ModelA)^2
  CV_MSE_B_ALT <- mean(rMSE_ModelB)^2
  
  ### print
  
  out <- data.frame(bias_A= CV_MSE_A - 1000,
                    bias_B= CV_MSE_B - 980,
                    bias_A_Alt= CV_MSE_A_ALT - 1000,
                    bias_B_Alt= CV_MSE_B_ALT - 980)
  
}

## 250,000 seed values

df_simulatedMSE <- future_map_dfr(.x=1:250000, .f=doOne, .options=future_options(seed=NA))

round(map_dbl(df_simulatedMSE, mean),2)

## bias_A     bias_B bias_A_Alt bias_B_Alt 
## 0.00       0.01      -0.08      -0.12