mse and ssim

[YangZhu327]

I am now training a model based on event reconstruction images, but when evaluating, MSE and SSIM have not been able to achieve the desired results.For example, when I reproduced E2VID, the values ​​of mse and SSIM did not reach the effect of the paper, and the gap was even very large.. I would like to ask what changes you made before calculating MSE and SSIM?

[ercanburak]

Hi. I'm not sure what is the exact issue that you are encountering, but you might want to inspect the code in this repository to see how MSE and SSIM are calculated after inference. You can find related code in eval.py and eval_metrics.py scripts. With the code in this EVREAL repository, using a command such as python eval.py -m E2VID -d ECD -qm mse ssim should give you the same MSE and SSIM results as in Table 2 of our paper (for E2VID on ECD dataset).

[YangZhu327]

Below is my evaluation code. I would like to know where my problem occurred and why I am unable to calculate a result that matches the paper.

`import argparse
import cv2
import glob
import numpy as np
from collections import OrderedDict
import os
import torch

import models
from src.utils_loss import LossFn, IntensityRescaler
from skimage.metrics import mean_squared_error as compare_mse
from skimage.metrics import structural_similarity as compare_ssim

def main():
parser = argparse.ArgumentParser()
parser.add_argument('--task', type=str, default='classical_sr', help='classical_sr, lightweight_sr, real_sr, '
'gray_dn, color_dn, jpeg_car, color_jpeg_car')
parser.add_argument('--folder_hr', type=str,
default='/media/njit5/39e16d05-05ba-46b4-bc9b-47b3c98f1d4f/zy/val/result/1',
help='input low-quality test image folder')
parser.add_argument('--folder_gt', type=str,
default='/media/njit5/39e16d05-05ba-46b4-bc9b-47b3c98f1d4f/zy/val/result/gt',
help='input ground-truth test image folder')
args = parser.parse_args()

torch.cuda.empty_cache()
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')

Lpips = models.PerceptualLoss(model='net-lin', net='alex', use_gpu=True)
# model.initialize(model='net-lin', net='alex', use_gpu=True)

Lpips.eval()
Lpips = Lpips.to(device)

save_dir = f'/media/njit5/39e16d05-05ba-46b4-bc9b-47b3c98f1d4f/zy/val/result'
os.makedirs(save_dir, exist_ok=True)

folder_gt = args.folder_gt

rescale = IntensityRescaler()
clahe = cv2.createCLAHE(clipLimit=2.0, tileGridSize=(8, 8))

test_results = OrderedDict()
test_results['mse'] = []
test_results['ssim'] = []
test_results['lpips'] = []
mse, ssim, lpips = 0, 0, 0

for idx, path in enumerate(sorted(glob.glob(os.path.join(folder_gt, '*')))):
# read image
imgname, img_lq, img_gt = get_image_pair(args, path) # image to HWC-BGR, float32

if img_gt is not None:

    img_gt = np.expand_dims(img_gt, axis=2)
    output = np.expand_dims(img_lq, axis=2)
    pred1 = torch.tensor(np.array(output)).permute(2, 0, 1).unsqueeze(0).float().to(device) / 255
    img1 = torch.tensor(np.array(img_gt)).permute(2, 0, 1).unsqueeze(0).float().to(device) / 255

    distance = Lpips(pred1, img1, normalize=True).mean()
    distance = distance.float()
    test_results['lpips'].append(distance)

    p = rescale(pred1)
    y = rescale(img1)

    p = p[0].detach().cpu().numpy().mean(0)
    y = y[0].detach().cpu().numpy().mean(0)

    p = np.uint8(cv2.normalize(p, None, 0, 255, cv2.NORM_MINMAX))
    y = np.uint8(cv2.normalize(y, None, 0, 255, cv2.NORM_MINMAX))

    y = clahe.apply(y)
    p = clahe.apply(p)

    ssim = compare_ssim(p, y, data_range=255, multichannel=False)
    mse = compare_mse(p / 255, y / 255)

    test_results['ssim'].append(ssim)
    test_results['mse'].append(mse)

    print('Testing {:d} {:20s} - SSIM: {:.6f}; MSE: {:.6f}; LPIPS: {:.4f};'.
          format(idx, imgname, ssim, mse, distance))`