Evaluate trained models on held-out data

Evaluate trained models on held-out data#

This notebook documents validation (main.py --validate) and provides step-by-step checks.

Checkpoint naming detail (from your current main.py):

  • --validate loads: {model_save_path}/{data_file}.pth

  • training saves: {model_save_path}/{run_name}.pth

0. Setup & parameters#

import os, sys, json, shutil
from pathlib import Path

project_root = Path.cwd()
os.chdir(project_root.parent.parent.parent)
project_root = Path.cwd()
print("project_root:", project_root)

DATA_FILE = "AUH_HepG2"
DATA_PATH = "./dataset"
TRANSFORMER_PATH = "./RBPformer"
RESULTS_DIR = "./results"
MODEL_SAVE_PATH = "./results/model"

SEED = 42
DEVICE_NUM = 1
USE_CPU = False
MAX_LENGTH = 101

print("DATA_FILE:", DATA_FILE)

project_root: /fs1/private/user/wangyubo/code/BRIDGE
DATA_FILE: AUH_HepG2

2. Rebuild the test split (print modality shapes)#

import random, torch
import numpy as np

def fix_seed(seed: int):
    torch.set_num_threads(1)
    random.seed(seed)
    os.environ["PYTHONHASHSEED"] = str(seed)
    np.random.seed(seed)
    torch.manual_seed(seed)
    torch.cuda.manual_seed(seed)
    torch.cuda.manual_seed_all(seed)

fix_seed(SEED)

device = torch.device("cpu") if USE_CPU else torch.device(f"cuda:{DEVICE_NUM}" if torch.cuda.is_available() else "cpu")
if device.type == "cuda":
    torch.cuda.set_device(DEVICE_NUM)
print("Device:", device)

Device: cuda:1

from utils.dataloaders import read_fasta
from utils.gen_transformer_embedding import build_Transformer_embeddings
from utils.structureFeatures import build_structure_tensor
from utils.FeatureEncoding import dealwithdata
from utils.motif_prior.motif_prior import get_motif_prior_matrix
from utils.utils import split_dataset, myDataset
from torch.utils.data import DataLoader

data_path = Path(DATA_PATH)
neg_fa = data_path / f"{DATA_FILE}_neg.fa"
pos_fa = data_path / f"{DATA_FILE}_pos.fa"
assert neg_fa.exists() and pos_fa.exists(), "Missing FASTA files."

sequences, structs, labels = read_fasta(str(neg_fa), str(pos_fa))
sequences, structs = list(sequences), list(structs)
labels = np.asarray(labels)

Transformer_emb, attention_weight = build_Transformer_embeddings(
    sequences=sequences,
    transformer_path=TRANSFORMER_PATH,
    device=device,
    k=1,
    transpose_to_ch_first=True,
)

structure = build_structure_tensor(structs, MAX_LENGTH)
biochem = dealwithdata(DATA_FILE).transpose([0, 2, 1])
motif = get_motif_prior_matrix(DATA_FILE)

print("Transformer_emb:", getattr(Transformer_emb, "shape", None))
print("attention_weight:", getattr(attention_weight, "shape", None))
print("structure:", getattr(structure, "shape", None))
print("biochem:", getattr(biochem, "shape", None))
print("motif:", getattr(motif, "shape", None))
print("labels:", labels.shape)

[INFO] Encoded 15002 sequences for AUH_HepG2, shape: (15002, 101, 99)
Valid motif file detected, skipping motif_prior: utils/motif_prior/output/AUH_HepG2/output/STRME_training_set.tab
Transformer_emb: (15002, 512, 101)
attention_weight: (15002, 101, 103)
structure: (15002, 1, 101)
biochem: (15002, 99, 101)
motif: (15002, 1, 24)
labels: (15002, 1)

(_, _, _, _, _, _), \
(test_emb, test_attn, test_struc, test_motif, test_biochem, test_label) = split_dataset(
    Transformer_emb,
    attention_weight,
    structure,
    motif,
    biochem,
    labels,
)

test_set = myDataset(test_emb, test_attn, test_struc, test_motif, test_biochem, test_label)
test_loader = DataLoader(test_set, batch_size=32*8, shuffle=False)

print("Test size:", len(test_set))

Test size: 3000

3. Load checkpoint and compute metrics#

import torch
import torch.nn as nn
from utils.BRIDGE import BRIDGE
from utils.train_loop import validate

model = BRIDGE().to(device)
model.load_state_dict(torch.load(dst, map_location=device))
model.eval()

criterion = nn.BCEWithLogitsLoss(pos_weight=torch.tensor(2).to(device))
met, y_all, p_all = validate(model, device, test_loader, criterion)

print("AUC  :", met.auc)
print("ACC  :", met.acc)
print("AUPRC:", met.prc)
print("MCC  :", met.mcc)

AUC  : 0.9020387500000001
ACC  : 0.8463333333333334
AUPRC: 0.8439708835663886
MCC  : 0.6440209826598539

4. Alternative: run main.py --validate (CLI)#

import subprocess, shlex

cmd = (
    f"python main.py "
    f"--validate "
    f"--data_file {DATA_FILE} "
    f"--data_path {DATA_PATH} "
    f"--Transformer_path {TRANSFORMER_PATH} "
    f"--model_save_path {MODEL_SAVE_PATH} "
    f"--seed {SEED} "
    f"--device_num {DEVICE_NUM}"
)

print("Command:\n", cmd)

# Uncomment to run:
# subprocess.run(shlex.split(cmd), check=True)

Command:
 python main.py --validate --data_file AUH_HepG2 --data_path ./dataset --Transformer_path ./RBPformer --model_save_path ./results/model --seed 42 --device_num 1

The Kernel crashed while executing code in the current cell or a previous cell. 

Please review the code in the cell(s) to identify a possible cause of the failure. 

Click <a href='https://aka.ms/vscodeJupyterKernelCrash'>here</a> for more info. 

View Jupyter <a href='command:jupyter.viewOutput'>log</a> for further details.
Contents