Compuational cost of models

This crude analysis summarizes the typical computational resources associated with each model. The results lack rigour, but accord with qualitative observations that heterogeneous models require substantially more time and memory to converge.

library("neotrans")
library("caret")
library("dplyr")

Simple summary statistics capture the headline trend, but because data are not universally available, include some biases:

db <- SlurmLog()
db <- db[which(db$State == "COMPLETED" & db$seconds > 60 &
               db$task == "marginal" & (db$inf == "ki") &
               db$pID %in% AllProjects()), ]
db$sPerCore <- db$seconds / db$nCPU
db$coreSeconds <- db$seconds * db$nCPU

tcoef <- summary(lm(log(seconds) ~ pID + model + nCPU, data = db))$coeff
tpcoef <- summary(lm(log(sPerCore) ~ pID + model, data = db))$coeff
tbycoef <- summary(lm(log(coreSeconds) ~ pID + model, data = db))$coeff
memcoef <- summary(lm(log(maxRSS) ~ pID + model, data = db))$coeff
modelRows <- rownames(tcoef)[startsWith(rownames(tcoef), "model")]
cbind(
  time = exp(tcoef[modelRows, "Estimate"]),
  tPerCore = exp(tpcoef[modelRows, "Estimate"]),
  coreSeconds = exp(tbycoef[modelRows, "Estimate"]),
  mem = exp(memcoef[modelRows, "Estimate"])
) |> `rownames<-`(ModelLabel(paste0(gsub("model", "", fixed = TRUE, modelRows), "_ki")))

##                    time  tPerCore coreSeconds       mem
## StN           1.0123724 1.0218283   0.9925451 0.6586688
## StN’          1.3970979 1.4791337   1.1630100 0.8713300
## StNT          1.2333995 1.4954045   1.0272247 0.8275082
## StT           1.2146017 1.4112706   1.0562238 0.8499770
## Het           0.6442466 0.7113299   0.5533979 0.9399860
## HetB          1.7979514 1.4439443   1.8337318 1.3206522
## HetBM         1.9323071 1.8392757   1.7618588 1.1248677
## HetM          1.9022444 1.7639777   1.7662068 1.1528526
## Het1P         1.8257724 1.4485544   1.8696305 1.3213777
## NstMk         1.1579098 1.4077777   0.9632911 0.8202848
## NstN          1.2043054 1.3538302   1.0557694 0.8642699
## NstNT         1.4385733 2.0069413   1.0332115 0.7127089
## NstT          1.2115251 1.6828533   0.8678912 0.7096459
## StN-shuffled  1.0052630 0.8486496   1.1298945 1.1211236
## StNT-shuffled 1.1873468 1.2724782   1.0834414 0.9049694
## StT-shuffled  1.1751000 1.2744805   1.0678330 0.8923388

It is more representative to explore how time and memory usage can be predicted after accounting for the datasets for which each model has resource data available:

train_control <- trainControl(method = "cv", number = 10)
model1_cv <- train(log(seconds) ~ pID + model + nCPU, data = db,
                   trControl = train_control, method = "lm")

## Warning in predict.lm(modelFit, newdata): prediction from rank-deficient fit;
## attr(*, "non-estim") has doubtful cases
## Warning in predict.lm(modelFit, newdata): prediction from rank-deficient fit;
## attr(*, "non-estim") has doubtful cases
## Warning in predict.lm(modelFit, newdata): prediction from rank-deficient fit;
## attr(*, "non-estim") has doubtful cases
## Warning in predict.lm(modelFit, newdata): prediction from rank-deficient fit;
## attr(*, "non-estim") has doubtful cases

model2_cv <- train(log(seconds) ~ pID + model, data = db, 
                   trControl = train_control, method = "lm")

## Warning in predict.lm(modelFit, newdata): prediction from rank-deficient fit;
## attr(*, "non-estim") has doubtful cases
## Warning in predict.lm(modelFit, newdata): prediction from rank-deficient fit;
## attr(*, "non-estim") has doubtful cases
## Warning in predict.lm(modelFit, newdata): prediction from rank-deficient fit;
## attr(*, "non-estim") has doubtful cases
## Warning in predict.lm(modelFit, newdata): prediction from rank-deficient fit;
## attr(*, "non-estim") has doubtful cases

cat(paste("Model 1 CV RMSE:", signif(model1_cv$results$RMSE)))

## Model 1 CV RMSE: 0.679761

cat(paste("Model 2 CV RMSE:", signif(model2_cv$results$RMSE)))

## Model 2 CV RMSE: 0.679016

has_by <- group_by(db, pID) |> 
  filter(any(model == "by")) |>
  ungroup() |>
  pull(pID)


norm <- db |>
  filter(pID %in% has_by) |>
  group_by(pID) |> 
  mutate(
    ref_s = sPerCore[model == "by"],
    ref_mem = maxRSS[model == "by"],
    norm_s = sPerCore / ref_s, norm_mem = maxRSS / ref_mem) |>
  ungroup()

norm |>
  group_by(scriptID) |>
  summarize(time = mean(norm_s), mem = mean(norm_mem), n = n()) |>
  mutate(model = ModelLabel(scriptID)) |>
  relocate(model, .before = scriptID) |>
  print(n = 100)

## # A tibble: 17 × 5
##    model         scriptID     time   mem     n
##    <chr>         <fct>       <dbl> <dbl> <int>
##  1 StMk          by_ki        1    1        48
##  2 StN           by_n_ki      1.21 0.989    42
##  3 StN’          by_nn_ki     2.58 1.27     45
##  4 StNT          by_nt_ki     1.95 1.000    46
##  5 StT           by_t_ki      6.18 0.905    44
##  6 HetB          hg_b_ki      3.11 1.97     29
##  7 HetBM         hg_bm_ki     3.58 1.79     28
##  8 Het           hg_ki        1.87 2.26     15
##  9 HetM          hg_m_ki      3.93 1.77     29
## 10 Het1P         hg2_ki       3.22 2.21     29
## 11 NstMk         ns_ki        1.84 0.942    44
## 12 NstN          ns_n_ki      1.92 1.05     46
## 13 NstNT         ns_nt_ki     3.26 0.701    36
## 14 NstT          ns_t_ki      2.39 0.900    45
## 15 StN-shuffled  rm_by_n_ki   1.05 1.15     42
## 16 StNT-shuffled rm_by_nt_ki  1.72 1.06     44
## 17 StT-shuffled  rm_by_t_ki   1.92 0.921    43

Martin R. Smith martin.smith@durham.ac.uk

2026-03-06