The R-value of Contrast-phys+ on UBFC-RPPG is low

[ZhiXinChasingLight]

Hello! Recently, I have been replicating Contrast-phys+ and following the steps to train and test it on UBFC-RPPG. The heart rate values are very close, but the calculated R-value always seems to be low. Do you know why? Here is the method I used to calculate the metrics:

subject:/home/zhixin_yan/project/contrast-phys/contrast-phys+/results/default/23/1/44.npy
hr_rppg:89.0
hr_bvp:89.0
MAE:0.7537042662656902
MSE:0.8300663503050997
RMSE:0.9110797716474116
R:0.5596756484461523

subject:/home/zhixin_yan/project/contrast-phys/contrast-phys+/results/default/23/1/49.npy
hr_rppg:87.0
hr_bvp:86.0
MAE:0.7199145203866735
MSE:0.7962145399996426
RMSE:0.8923085452911692
R:0.5253803322031185

subject:/home/zhixin_yan/project/contrast-phys/contrast-phys+/results/default/23/1/45.npy
hr_rppg:112.0
hr_bvp:112.0
MAE:0.7285037279791446
MSE:0.7469296112617089
RMSE:0.8642508960144091
R:0.7825049275582626

subject:/home/zhixin_yan/project/contrast-phys/contrast-phys+/results/default/23/1/46.npy
hr_rppg:94.00000000000001
hr_bvp:93.0
MAE:0.7080922650721068
MSE:0.7148581794156879
RMSE:0.8454928618360346
R:0.44332718968619483

subject:/home/zhixin_yan/project/contrast-phys/contrast-phys+/results/default/23/1/39.npy
hr_rppg:86.0
hr_bvp:86.0
MAE:0.6574607328649096
MSE:0.707239877504444
RMSE:0.8409755510741344
R:0.5742140765121817

subject:/home/zhixin_yan/project/contrast-phys/contrast-phys+/results/default/23/1/43.npy
hr_rppg:99.99999999999999
hr_bvp:99.99999999999999
MAE:0.7248150684600015
MSE:0.7970786218834163
RMSE:0.892792597350256
R:0.6297846759769277

subject:/home/zhixin_yan/project/contrast-phys/contrast-phys+/results/default/23/1/38.npy
hr_rppg:112.0
hr_bvp:112.0
MAE:0.7259880192697764
MSE:0.7252420109154502
RMSE:0.8516114201415163
R:0.8895395112121949

subject:/home/zhixin_yan/project/contrast-phys/contrast-phys+/results/default/23/1/42.npy
hr_rppg:98.0
hr_bvp:98.0
MAE:0.715046724078873
MSE:0.7930097950294874
RMSE:0.8905109741207502
R:0.7976646616673656

subject:/home/zhixin_yan/project/contrast-phys/contrast-phys+/results/default/23/1/40.npy
hr_rppg:88.0
hr_bvp:88.0
MAE:0.6873482923836377
MSE:0.7130528581768157
RMSE:0.8444245722246693
R:0.5340293702773984

subject:/home/zhixin_yan/project/contrast-phys/contrast-phys+/results/default/23/1/41.npy
hr_rppg:99.99999999999999
hr_bvp:99.99999999999999
MAE:0.6872299273870676
MSE:0.676695129281339
RMSE:0.822614812218537
R:0.59450306089585

subject:/home/zhixin_yan/project/contrast-phys/contrast-phys+/results/default/23/1/47.npy
hr_rppg:117.00000000000001
hr_bvp:117.00000000000001
MAE:0.6963591330861216
MSE:0.6587551545136888
RMSE:0.8116373294234863
R:0.6429711341256403

subject:/home/zhixin_yan/project/contrast-phys/contrast-phys+/results/default/23/1/48.npy
hr_rppg:92.0
hr_bvp:92.0
MAE:0.803484226944761
MSE:0.8820074963485776
RMSE:0.9391525415759558
R:0.3685707944758343
MAE:0.7173289086815637
MSE:0.7534291353862799
RMSE:0.8680029581667794
R:0.5585138808441661

[zhaodongsun]

hi, please see the issue #16

[ZhiXinChasingLight]

hi, please see the issue #16
Hello, thank you for your answer. Your answer has solved my problem well, but I still can't reproduce the good effect in the paper on the UBFC-RPPG data set. Is this a problem of parameter selection? I have provided the duplicate result of heart rate estimation on UBFC-RPPG and the code of train.py. Could you please help me have a look?

import cv2
import matplotlib.pyplot as plt
import numpy as np
import os
import h5py
import torch
from PhysNetModel import PhysNet
from loss import ContrastLoss
from IrrelevantPowerRatio import IrrelevantPowerRatio

from utils_data import *
from utils_sig import *
from torch import optim
from torch.utils.data import DataLoader
from sacred import Experiment
from sacred.observers import FileStorageObserver

import matplotlib.pyplot as plt

ex = Experiment('model_train', save_git_info=False)

if torch.cuda.is_available():
device = torch.device('cuda')
torch.backends.cudnn.enabled = True
torch.backends.cudnn.benchmark = True
else:
device = torch.device('cpu')

@ex.config
def my_config():
# here are some hyperparameters in our method

# hyperparams for model training
total_epoch = 30 # total number of epochs for training the model
lr = 1e-5 # learning rate
in_ch = 3 # TODO: number of input video channels, in_ch=3 for RGB videos, in_ch=1 for NIR videos.

# hyperparams for ST-rPPG block
fs = 30 # video frame rate, TODO: modify it if your video frame rate is not 30 fps.
T = fs * 10 # temporal dimension of ST-rPPG block, default is 10 seconds.
S = 2 # spatial dimenion of ST-rPPG block, default is 2x2.

# hyperparams for rPPG spatiotemporal sampling
delta_t = int(T/2) # time length of each rPPG sample
K = 4 # the number of rPPG samples at each spatial position

label_ratio = 1 # TODO: if you dataset is fully labeled, you can set how many labels are used for training.

train_exp_name = 'default'
result_dir = './results/%s'%(train_exp_name) # store checkpoints and training recording
os.makedirs(result_dir, exist_ok=True)
ex.observers.append(FileStorageObserver(result_dir))

@ex.automain
def my_main(_run, total_epoch, T, S, lr, result_dir, fs, delta_t, K, in_ch, label_ratio):

exp_dir = result_dir + '/%d'%(int(_run._id)) # store experiment recording to the path

# get the training and test file path list by spliting the dataset
train_list, test_list = UBFC_LU_split() # TODO: you should define your function to split your dataset for training and testing
np.save(exp_dir+'/train_list.npy', train_list)
np.save(exp_dir+'/test_list.npy', test_list)

# define the dataloader
dataset = H5Dataset(train_list, T, label_ratio) # please read the code about H5Dataset when preparing your dataset
dataloader = DataLoader(dataset, batch_size=2, # two videos for contrastive learning
                        shuffle=True, num_workers=4, pin_memory=True, drop_last=True)

# define the model and loss
model = PhysNet(S, in_ch=in_ch).to(device).train()
loss_func = ContrastLoss(delta_t, K, fs, high_pass=40, low_pass=250)

# define irrelevant power ratio
IPR = IrrelevantPowerRatio(Fs=fs, high_pass=40, low_pass=250)

# define the optimizer
opt = optim.AdamW(model.parameters(), lr=lr)

total_epochs = []  # 存储每个 epoch 的值
loss_mean_values = []  # 存储损失平均值

for e in range(total_epoch):
    epoch_loss_values = []  # 存储当前 epoch 的损失值
    for it in range(np.round(60/(T/fs)).astype('int')): # TODO: 60 means the video length of each video is 60s. If each video's length in your dataset is other value (e.g, 30s), you should use that value.
        for imgs, GT_sig, label_flag in dataloader: # dataloader randomly samples a video clip with length T
            imgs = imgs.to(device)
            GT_sig = GT_sig.to(device) # input: shape (2, T),但现在为[2, 300, 1]不太对劲
            GT_sig = GT_sig.squeeze(dim=2) # 去除第三个维度
            label_flag = label_flag.to(device)
            # import pdb; pdb.set_trace()
            # model forward propagation
            model_output = model(imgs) 
            rppg = model_output[:,-1] # get rppg
            # import pdb; pdb.set_trace()
            # define the loss functions
            loss, p_loss, n_loss, p_loss_gt, n_loss_gt = loss_func(model_output, GT_sig, label_flag)

            # optimize
            opt.zero_grad()
            loss.backward()
            opt.step()

            # 保存损失值
            loss_value = loss.item()
            epoch_loss_values.append(loss_value)

            # evaluate irrelevant power ratio during training
            ipr = torch.mean(IPR(rppg.clone().detach()))

            # save loss values and IPR
            ex.log_scalar("loss", loss.item())
            ex.log_scalar("p_loss", p_loss.item())
            ex.log_scalar("n_loss", n_loss.item())
            ex.log_scalar("p_loss_gt", p_loss_gt.item())
            ex.log_scalar("n_loss_gt", n_loss_gt.item())
            ex.log_scalar("ipr", ipr.item())

    # 计算当前 epoch 的损失平均值
    epoch_loss_mean = sum(epoch_loss_values) / len(epoch_loss_values)
    loss_mean_values.append(epoch_loss_mean)
    total_epochs.append(e)
    print(f"epoch{e+1}/{total_epoch}:{epoch_loss_mean:.4f}")
    # save model checkpoints
    torch.save(model.state_dict(), exp_dir+'/epoch%d.pt'%e)

# 绘制折线图
plt.plot(total_epochs, loss_mean_values)
plt.xlabel('Epoch')
plt.ylabel('Loss Mean')
plt.title('Loss Mean vs Epoch')
plt.savefig(f"contrast-phys+_loss_{label_ratio}.png",dpi=500)
plt.show()