Section 5 Performance evaluation

library(stringr)
library(DescTools)
library(pROC)
library(MLmetrics)
load("./rdata/train_test_splits_PD_PDism.RData")
data$response = factor(ifelse(data$response == 1, "PD", "PDism"))

5.1 All tasks

num_splits = 5
all_sl_preds = data.frame(matrix(nrow = nrow(data), ncol = 0))
for (i in 1:num_splits) {
    sl_preds = read.csv(paste0("./files/split", i, "_PD_PDism_miee_30.csv"))
    sl_preds <- sl_preds[order(sl_preds$PDGP), ]
    sl_preds = subset(sl_preds, select = -PDGP)
    colnames(sl_preds) = paste0(colnames(sl_preds), "_split",
        i)
    all_sl_preds = cbind(all_sl_preds, sl_preds)
}

all_sl_pred_cls = all_sl_preds
all_sl_pred_cls$predict_mode = apply(all_sl_pred_cls, 1, function(x) {
    uniqx <- unique(na.omit(x))
    uniqx[which.max(tabulate(match(x, uniqx)))]
})

all_sl_pred_cls$predict_mode = ifelse(all_sl_pred_cls$predict_mode ==
    0, "PD", "PDism")

conf = caret::confusionMatrix(data = factor(all_sl_pred_cls$predict_mode,
    levels = c("PD", "PDism")), factor(data$response, levels = c("PD",
    "PDism")))
knitr::kable(conf$table)

	PD	PDism
PD	177	4
PDism	83	14

cat("Balanced accuracy = ", round(conf[["byClass"]][["Balanced Accuracy"]] *
    100, 2), "%\n")

## Balanced accuracy =  72.93 %

cat("F1_score = ", F1_Score(factor(all_sl_pred_cls$predict_mode,
    levels = c("PD", "PDism")), factor(data$response, levels = c("PD",
    "PDism"))), "\n")

## F1_score =  0.8027211

auc = auc(as.numeric(data$response), as.numeric(factor(all_sl_pred_cls$predict_mode)))
print(auc)

## Area under the curve: 0.7293

5.2 TUG-only

num_splits = 5
all_sl_preds = data.frame(matrix(nrow = nrow(data), ncol = 0))
for (i in 1:num_splits) {
    sl_preds = read.csv(paste0("./files/split", i, "_PD_PDism_miee_tug_30.csv"))
    sl_preds <- sl_preds[order(sl_preds$PDGP), ]
    sl_preds = subset(sl_preds, select = -PDGP)
    colnames(sl_preds) = paste0(colnames(sl_preds), "_split",
        i)
    all_sl_preds = cbind(all_sl_preds, sl_preds)
}

all_sl_pred_cls = all_sl_preds
all_sl_pred_cls$predict_mode = apply(all_sl_pred_cls, 1, function(x) {
    uniqx <- unique(na.omit(x))
    uniqx[which.max(tabulate(match(x, uniqx)))]
})

all_sl_pred_cls$predict_mode = ifelse(all_sl_pred_cls$predict_mode ==
    0, "PD", "PDism")

conf = caret::confusionMatrix(data = factor(all_sl_pred_cls$predict_mode,
    levels = c("PD", "PDism")), factor(data$response, levels = c("PD",
    "PDism")))
knitr::kable(conf$table)

	PD	PDism
PD	190	3
PDism	70	15

cat("Balanced accuracy = ", round(conf[["byClass"]][["Balanced Accuracy"]] *
    100, 2), "%\n")

## Balanced accuracy =  78.21 %

cat("F1_score = ", F1_Score(factor(all_sl_pred_cls$predict_mode,
    levels = c("PD", "PDism")), factor(data$response, levels = c("PD",
    "PDism"))), "\n")

## F1_score =  0.8388521

auc = auc(as.numeric(data$response), as.numeric(factor(all_sl_pred_cls$predict_mode)))
print(auc)

## Area under the curve: 0.7821