Incorrect hyperparameter for ranger.

Question

Incorrect hyperparameter for ranger.

Closed this issue 2 years ago · 2 comments

Thanks for this package, it is very helpful. One point to note:

It looks like the minimum value reset in line 183 of ffs should be .mtry instead of mtry.

When running the example within the ffs function with method = ranger, it is clear that the instances with mtry > # of predictors isn't skipped in the same way as when the tuneLength argument is set. See example below.

library(CAST) 
#> Warning: package 'CAST' was built under R version 4.1.3
library(doParallel)
#> Warning: package 'doParallel' was built under R version 4.1.3
#> Loading required package: foreach
#> Warning: package 'foreach' was built under R version 4.1.3
#> Loading required package: iterators
#> Warning: package 'iterators' was built under R version 4.1.3
#> Loading required package: parallel
library(lubridate)
#> Warning: package 'lubridate' was built under R version 4.1.2
#> 
#> Attaching package: 'lubridate'
#> The following objects are masked from 'package:base':
#> 
#>     date, intersect, setdiff, union
library(ggplot2)
#> Warning: package 'ggplot2' was built under R version 4.1.2
library(caret)
#> Warning: package 'caret' was built under R version 4.1.3
#> Loading required package: lattice
#> Warning: package 'lattice' was built under R version 4.1.2
library(tictoc)
#> Warning: package 'tictoc' was built under R version 4.1.1
 cl <- makeCluster(3)
 registerDoParallel(cl)

 #load and prepare dataset:
 dat <- get(load(system.file("extdata","Cookfarm.RData",package="CAST")))
 trainDat <- dat[dat$altitude==-0.3&year(dat$Date)==2012&week(dat$Date)%in%c(13:14),]

 #visualize dataset:
 ggplot(data = trainDat, aes(x=Date, y=VW)) + geom_line(aes(colour=SOURCEID))

 #create folds for Leave Location Out Cross Validation:
 set.seed(10)
 indices <- CreateSpacetimeFolds(trainDat,spacevar = "SOURCEID",k=3)
 ctrl <- trainControl(method="cv",index = indices$index)

 #define potential predictors:
 predictors <- c("DEM","TWI","BLD","Precip_cum","cday","MaxT_wrcc",
 "Precip_wrcc","NDRE.M","Bt","MinT_wrcc","Northing","Easting")
 
 tuneGrid<- data.frame(
   .mtry = c(2,5,7),
   .splitrule = "variance",
   .min.node.size = c(5))

 tic()
 
 #run ffs model with Leave Location out CV
 set.seed(10)
 ffsmodel <- ffs(trainDat[,predictors],trainDat$VW,method="ranger",
 tuneGrid = tuneGrid, num.trees = 100,trControl=ctrl)
#> [1] "model using DEM,TWI will be trained now..."
#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> There were missing values in resampled performance measures.
#> Warning in train.default(predictors[, minGrid[i, ]], response, method =
#> method, : missing values found in aggregated results
#> [1] "maximum number of models that still need to be trained: 120"
#> [1] "model using DEM,BLD will be trained now..."
#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> There were missing values in resampled performance measures.

#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> missing values found in aggregated results
#> [1] "maximum number of models that still need to be trained: 119"
#> [1] "model using DEM,Precip_cum will be trained now..."
#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> There were missing values in resampled performance measures.

#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> missing values found in aggregated results
#> [1] "maximum number of models that still need to be trained: 118"
#> [1] "model using DEM,cday will be trained now..."
#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> There were missing values in resampled performance measures.

#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> missing values found in aggregated results
#> [1] "maximum number of models that still need to be trained: 117"
#> [1] "model using DEM,MaxT_wrcc will be trained now..."
#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> There were missing values in resampled performance measures.

#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> missing values found in aggregated results
#> [1] "maximum number of models that still need to be trained: 116"
#> [1] "model using DEM,Precip_wrcc will be trained now..."
#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> There were missing values in resampled performance measures.

#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> missing values found in aggregated results
#> [1] "maximum number of models that still need to be trained: 115"
#> [1] "model using DEM,NDRE.M will be trained now..."
#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> There were missing values in resampled performance measures.

#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> missing values found in aggregated results
#> [1] "maximum number of models that still need to be trained: 114"
#> [1] "model using DEM,Bt will be trained now..."
#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> There were missing values in resampled performance measures.

#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> missing values found in aggregated results
#> [1] "maximum number of models that still need to be trained: 113"
#> [1] "model using DEM,MinT_wrcc will be trained now..."
#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> There were missing values in resampled performance measures.

#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> missing values found in aggregated results
#> [1] "maximum number of models that still need to be trained: 112"
#> [1] "model using DEM,Northing will be trained now..."
#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> There were missing values in resampled performance measures.

#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> missing values found in aggregated results
#> [1] "maximum number of models that still need to be trained: 111"
#> [1] "model using DEM,Easting will be trained now..."
#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> There were missing values in resampled performance measures.

#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> missing values found in aggregated results
#> [1] "maximum number of models that still need to be trained: 110"
#> [1] "model using TWI,BLD will be trained now..."
#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> There were missing values in resampled performance measures.

#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> missing values found in aggregated results
#> [1] "maximum number of models that still need to be trained: 109"
#> [1] "model using TWI,Precip_cum will be trained now..."
#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> There were missing values in resampled performance measures.

#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> missing values found in aggregated results
#> [1] "maximum number of models that still need to be trained: 108"
#> [1] "model using TWI,cday will be trained now..."
#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> There were missing values in resampled performance measures.

#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> missing values found in aggregated results
#> [1] "maximum number of models that still need to be trained: 107"
#> [1] "model using TWI,MaxT_wrcc will be trained now..."
#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> There were missing values in resampled performance measures.

#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> missing values found in aggregated results
#> [1] "maximum number of models that still need to be trained: 106"
#> [1] "model using TWI,Precip_wrcc will be trained now..."
#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> There were missing values in resampled performance measures.

#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> missing values found in aggregated results
#> [1] "maximum number of models that still need to be trained: 105"
#> [1] "model using TWI,NDRE.M will be trained now..."
#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> There were missing values in resampled performance measures.

#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> missing values found in aggregated results
#> [1] "maximum number of models that still need to be trained: 104"
#> [1] "model using TWI,Bt will be trained now..."
#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> There were missing values in resampled performance measures.

#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> missing values found in aggregated results
#> [1] "maximum number of models that still need to be trained: 103"
#> [1] "model using TWI,MinT_wrcc will be trained now..."
#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> There were missing values in resampled performance measures.

#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> missing values found in aggregated results
#> [1] "maximum number of models that still need to be trained: 102"
#> [1] "model using TWI,Northing will be trained now..."
#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> There were missing values in resampled performance measures.

#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> missing values found in aggregated results
#> [1] "maximum number of models that still need to be trained: 101"
#> [1] "model using TWI,Easting will be trained now..."
#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> There were missing values in resampled performance measures.

#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> missing values found in aggregated results
#> [1] "maximum number of models that still need to be trained: 100"
#> [1] "model using BLD,Precip_cum will be trained now..."
#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> There were missing values in resampled performance measures.

#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> missing values found in aggregated results
#> [1] "maximum number of models that still need to be trained: 99"
#> [1] "model using BLD,cday will be trained now..."
#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> There were missing values in resampled performance measures.

#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> missing values found in aggregated results
#> [1] "maximum number of models that still need to be trained: 98"
#> [1] "model using BLD,MaxT_wrcc will be trained now..."
#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> There were missing values in resampled performance measures.

#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> missing values found in aggregated results
#> [1] "maximum number of models that still need to be trained: 97"
#> [1] "model using BLD,Precip_wrcc will be trained now..."
#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> There were missing values in resampled performance measures.

#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> missing values found in aggregated results
#> [1] "maximum number of models that still need to be trained: 96"
#> [1] "model using BLD,NDRE.M will be trained now..."
#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> There were missing values in resampled performance measures.

#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> missing values found in aggregated results
#> [1] "maximum number of models that still need to be trained: 95"
#> [1] "model using BLD,Bt will be trained now..."
#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> There were missing values in resampled performance measures.

#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> missing values found in aggregated results
#> [1] "maximum number of models that still need to be trained: 94"
#> [1] "model using BLD,MinT_wrcc will be trained now..."
#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> There were missing values in resampled performance measures.

#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> missing values found in aggregated results
#> [1] "maximum number of models that still need to be trained: 93"
#> [1] "model using BLD,Northing will be trained now..."
#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> There were missing values in resampled performance measures.

#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> missing values found in aggregated results
#> [1] "maximum number of models that still need to be trained: 92"
#> [1] "model using BLD,Easting will be trained now..."
#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> There were missing values in resampled performance measures.

#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> missing values found in aggregated results
#> [1] "maximum number of models that still need to be trained: 91"
#> [1] "model using Precip_cum,cday will be trained now..."
#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> There were missing values in resampled performance measures.

#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> missing values found in aggregated results
#> [1] "maximum number of models that still need to be trained: 90"
#> [1] "model using Precip_cum,MaxT_wrcc will be trained now..."
#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> There were missing values in resampled performance measures.

#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> missing values found in aggregated results
#> [1] "maximum number of models that still need to be trained: 89"
#> [1] "model using Precip_cum,Precip_wrcc will be trained now..."
#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> There were missing values in resampled performance measures.

#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> missing values found in aggregated results
#> [1] "maximum number of models that still need to be trained: 88"
#> [1] "model using Precip_cum,NDRE.M will be trained now..."
#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> There were missing values in resampled performance measures.

#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> missing values found in aggregated results
#> [1] "maximum number of models that still need to be trained: 87"
#> [1] "model using Precip_cum,Bt will be trained now..."
#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> There were missing values in resampled performance measures.

#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> missing values found in aggregated results
#> [1] "maximum number of models that still need to be trained: 86"
#> [1] "model using Precip_cum,MinT_wrcc will be trained now..."
#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> There were missing values in resampled performance measures.

#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> missing values found in aggregated results
#> [1] "maximum number of models that still need to be trained: 85"
#> [1] "model using Precip_cum,Northing will be trained now..."
#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> There were missing values in resampled performance measures.

#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> missing values found in aggregated results
#> [1] "maximum number of models that still need to be trained: 84"
#> [1] "model using Precip_cum,Easting will be trained now..."
#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> There were missing values in resampled performance measures.

#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> missing values found in aggregated results
#> [1] "maximum number of models that still need to be trained: 83"
#> [1] "model using cday,MaxT_wrcc will be trained now..."
#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> There were missing values in resampled performance measures.

#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> missing values found in aggregated results
#> [1] "maximum number of models that still need to be trained: 82"
#> [1] "model using cday,Precip_wrcc will be trained now..."
#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> There were missing values in resampled performance measures.

#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> missing values found in aggregated results
#> [1] "maximum number of models that still need to be trained: 81"
#> [1] "model using cday,NDRE.M will be trained now..."
#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> There were missing values in resampled performance measures.

#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> missing values found in aggregated results
#> [1] "maximum number of models that still need to be trained: 80"
#> [1] "model using cday,Bt will be trained now..."
#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> There were missing values in resampled performance measures.

#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> missing values found in aggregated results
#> [1] "maximum number of models that still need to be trained: 79"
#> [1] "model using cday,MinT_wrcc will be trained now..."
#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> There were missing values in resampled performance measures.

#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> missing values found in aggregated results
#> [1] "maximum number of models that still need to be trained: 78"
#> [1] "model using cday,Northing will be trained now..."
#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> There were missing values in resampled performance measures.

#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> missing values found in aggregated results
#> [1] "maximum number of models that still need to be trained: 77"
#> [1] "model using cday,Easting will be trained now..."
#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> There were missing values in resampled performance measures.

#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> missing values found in aggregated results
#> [1] "maximum number of models that still need to be trained: 76"
#> [1] "model using MaxT_wrcc,Precip_wrcc will be trained now..."
#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> There were missing values in resampled performance measures.

#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> missing values found in aggregated results
#> [1] "maximum number of models that still need to be trained: 75"
#> [1] "model using MaxT_wrcc,NDRE.M will be trained now..."
#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> There were missing values in resampled performance measures.

#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> missing values found in aggregated results
#> [1] "maximum number of models that still need to be trained: 74"
#> [1] "model using MaxT_wrcc,Bt will be trained now..."
#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> There were missing values in resampled performance measures.

#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> missing values found in aggregated results
#> [1] "maximum number of models that still need to be trained: 73"
#> [1] "model using MaxT_wrcc,MinT_wrcc will be trained now..."
#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> There were missing values in resampled performance measures.

#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> missing values found in aggregated results
#> [1] "maximum number of models that still need to be trained: 72"
#> [1] "model using MaxT_wrcc,Northing will be trained now..."
#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> There were missing values in resampled performance measures.

#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> missing values found in aggregated results
#> [1] "maximum number of models that still need to be trained: 71"
#> [1] "model using MaxT_wrcc,Easting will be trained now..."
#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> There were missing values in resampled performance measures.

#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> missing values found in aggregated results
#> [1] "maximum number of models that still need to be trained: 70"
#> [1] "model using Precip_wrcc,NDRE.M will be trained now..."
#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> There were missing values in resampled performance measures.

#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> missing values found in aggregated results
#> [1] "maximum number of models that still need to be trained: 69"
#> [1] "model using Precip_wrcc,Bt will be trained now..."
#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> There were missing values in resampled performance measures.

#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> missing values found in aggregated results
#> [1] "maximum number of models that still need to be trained: 68"
#> [1] "model using Precip_wrcc,MinT_wrcc will be trained now..."
#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> There were missing values in resampled performance measures.

#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> missing values found in aggregated results
#> [1] "maximum number of models that still need to be trained: 67"
#> [1] "model using Precip_wrcc,Northing will be trained now..."
#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> There were missing values in resampled performance measures.

#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> missing values found in aggregated results
#> [1] "maximum number of models that still need to be trained: 66"
#> [1] "model using Precip_wrcc,Easting will be trained now..."
#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> There were missing values in resampled performance measures.

#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> missing values found in aggregated results
#> [1] "maximum number of models that still need to be trained: 65"
#> [1] "model using NDRE.M,Bt will be trained now..."
#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> There were missing values in resampled performance measures.

#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> missing values found in aggregated results
#> [1] "maximum number of models that still need to be trained: 64"
#> [1] "model using NDRE.M,MinT_wrcc will be trained now..."
#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> There were missing values in resampled performance measures.

#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> missing values found in aggregated results
#> [1] "maximum number of models that still need to be trained: 63"
#> [1] "model using NDRE.M,Northing will be trained now..."
#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> There were missing values in resampled performance measures.

#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> missing values found in aggregated results
#> [1] "maximum number of models that still need to be trained: 62"
#> [1] "model using NDRE.M,Easting will be trained now..."
#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> There were missing values in resampled performance measures.

#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> missing values found in aggregated results
#> [1] "maximum number of models that still need to be trained: 61"
#> [1] "model using Bt,MinT_wrcc will be trained now..."
#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> There were missing values in resampled performance measures.

#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> missing values found in aggregated results
#> [1] "maximum number of models that still need to be trained: 60"
#> [1] "model using Bt,Northing will be trained now..."
#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> There were missing values in resampled performance measures.

#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> missing values found in aggregated results
#> [1] "maximum number of models that still need to be trained: 59"
#> [1] "model using Bt,Easting will be trained now..."
#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> There were missing values in resampled performance measures.

#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> missing values found in aggregated results
#> [1] "maximum number of models that still need to be trained: 58"
#> [1] "model using MinT_wrcc,Northing will be trained now..."
#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> There were missing values in resampled performance measures.

#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> missing values found in aggregated results
#> [1] "maximum number of models that still need to be trained: 57"
#> [1] "model using MinT_wrcc,Easting will be trained now..."
#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> There were missing values in resampled performance measures.

#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> missing values found in aggregated results
#> [1] "maximum number of models that still need to be trained: 56"
#> [1] "model using Northing,Easting will be trained now..."
#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> There were missing values in resampled performance measures.

#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> missing values found in aggregated results
#> [1] "maximum number of models that still need to be trained: 55"
#> [1] "vars selected: DEM,BLD with RMSE NaN"
#> [1] "model using additional variable TWI will be trained now..."
#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> There were missing values in resampled performance measures.
#> Warning in train.default(predictors[, c(startvars, nextvars[i])], response, :
#> missing values found in aggregated results
#> [1] "maximum number of models that still need to be trained: 54"
#> [1] "model using additional variable Precip_cum will be trained now..."
#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> There were missing values in resampled performance measures.

#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> missing values found in aggregated results
#> [1] "maximum number of models that still need to be trained: 53"
#> [1] "model using additional variable cday will be trained now..."
#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> There were missing values in resampled performance measures.

#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> missing values found in aggregated results
#> [1] "maximum number of models that still need to be trained: 52"
#> [1] "model using additional variable MaxT_wrcc will be trained now..."
#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> There were missing values in resampled performance measures.

#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> missing values found in aggregated results
#> [1] "maximum number of models that still need to be trained: 51"
#> [1] "model using additional variable Precip_wrcc will be trained now..."
#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> There were missing values in resampled performance measures.

#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> missing values found in aggregated results
#> [1] "maximum number of models that still need to be trained: 50"
#> [1] "model using additional variable NDRE.M will be trained now..."
#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> There were missing values in resampled performance measures.

#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> missing values found in aggregated results
#> [1] "maximum number of models that still need to be trained: 49"
#> [1] "model using additional variable Bt will be trained now..."
#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> There were missing values in resampled performance measures.

#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> missing values found in aggregated results
#> [1] "maximum number of models that still need to be trained: 48"
#> [1] "model using additional variable MinT_wrcc will be trained now..."
#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> There were missing values in resampled performance measures.

#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> missing values found in aggregated results
#> [1] "maximum number of models that still need to be trained: 47"
#> [1] "model using additional variable Northing will be trained now..."
#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> There were missing values in resampled performance measures.

#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> missing values found in aggregated results
#> [1] "maximum number of models that still need to be trained: 46"
#> [1] "model using additional variable Easting will be trained now..."
#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> There were missing values in resampled performance measures.

#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> missing values found in aggregated results
#> [1] "maximum number of models that still need to be trained: 45"
#> [1] "vars selected: DEM,BLD,Northing with RMSE NaN"
#> [1] "model using additional variable TWI will be trained now..."
#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> There were missing values in resampled performance measures.

#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> missing values found in aggregated results
#> [1] "maximum number of models that still need to be trained: 44"
#> [1] "model using additional variable Precip_cum will be trained now..."
#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> There were missing values in resampled performance measures.

#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> missing values found in aggregated results
#> [1] "maximum number of models that still need to be trained: 43"
#> [1] "model using additional variable cday will be trained now..."
#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> There were missing values in resampled performance measures.

#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> missing values found in aggregated results
#> [1] "maximum number of models that still need to be trained: 42"
#> [1] "model using additional variable MaxT_wrcc will be trained now..."
#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> There were missing values in resampled performance measures.

#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> missing values found in aggregated results
#> [1] "maximum number of models that still need to be trained: 41"
#> [1] "model using additional variable Precip_wrcc will be trained now..."
#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> There were missing values in resampled performance measures.

#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> missing values found in aggregated results
#> [1] "maximum number of models that still need to be trained: 40"
#> [1] "model using additional variable NDRE.M will be trained now..."
#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> There were missing values in resampled performance measures.

#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> missing values found in aggregated results
#> [1] "maximum number of models that still need to be trained: 39"
#> [1] "model using additional variable Bt will be trained now..."
#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> There were missing values in resampled performance measures.

#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> missing values found in aggregated results
#> [1] "maximum number of models that still need to be trained: 38"
#> [1] "model using additional variable MinT_wrcc will be trained now..."
#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> There were missing values in resampled performance measures.

#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> missing values found in aggregated results
#> [1] "maximum number of models that still need to be trained: 37"
#> [1] "model using additional variable Easting will be trained now..."
#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> There were missing values in resampled performance measures.

#> Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
#> missing values found in aggregated results
#> [1] "maximum number of models that still need to be trained: 36"
#> [1] "vars selected: DEM,BLD,Northing with RMSE NaN"
#> Note: No increase in performance found using more than 3 variables
 ffsmodel
#> Random Forest 
#> 
#> 490 samples
#>   3 predictor
#> 
#> No pre-processing
#> Resampling: Cross-Validated (10 fold) 
#> Summary of sample sizes: 332, 326, 322 
#> Resampling results across tuning parameters:
#> 
#>   mtry  RMSE        Rsquared   MAE       
#>   2     0.06282679  0.2913655  0.04683066
#>   5            NaN        NaN         NaN
#>   7            NaN        NaN         NaN
#> 
#> Tuning parameter 'splitrule' was held constant at a value of variance
#> 
#> Tuning parameter 'min.node.size' was held constant at a value of 5
#> RMSE was used to select the optimal model using the smallest value.
#> The final values used for the model were mtry = 2, splitrule = variance
#>  and min.node.size = 5.

 toc()
#> 36.04 sec elapsed

 stopCluster(cl)

^{Created on 2022-04-24 by the reprex package (v2.0.1)}

Answer 1 · 2022-04-27T10:02:21.000Z

Dear @pgoodling-usgs ,
thank you for your feedback!
Could you rerun your code with tuneGrid without the dots?

tuneGrid<- data.frame(
mtry = c(2,5,7),
splitrule = "variance",
min.node.size = c(5))

The current behavior is that mtry is set to the maximum number of predictors in each round of the ffs - so I guess it works as intended?

Greetings, Marvin

Answer 2 · 2022-04-27T12:35:40.000Z

Hi @Ludwigm6,

Sorry, you are correct! I was misunderstanding how the mtry gets fed into the ffs function - I didn't realize that it would rename it automatically.

Best,

Phillip