PR-curve with the complete negative test sample without a salient object inside

[songkq]

Greeting!
If there are some test samples without a salient object in the image, does it need to contain these samples when computing the PR-curve? Since the precision and recall keep zero with any threshold.

[ArcherFMY]

Hi,
If you mean some cases in PASCAL-S dataset, I would like to exclude these images to compute the precision and recall for this dataset.

[songkq]

@ArcherFMY I think those samples without salient objects are also important for evaluating the model. So why not to directly compute the measures from the whole dataset instead of obtaining the average mesures? Any advice?

[ArcherFMY]

@ipScore I think it is OK if you evaluate all the models using the same set of imgs&gts.

[songkq]

@ArcherFMY I directly calculate the PR mesures from the whole dataset as the following based on your code, which avoids the ill definition formulation for those samples without the salient objects. I think it works well.

%%
function Metrics = ...
    performCalcuUpdate(datasetStruct,algStructArray) 

evaluateSal = @(sMap,thresholds,gtMap) thresholdBased_HR_FR(sMap,thresholds,gtMap);
compute_TP_FP = @(sMap,thresholds,gtMap) thresholdBased_TP_FP(sMap,thresholds,gtMap);

thresholds = [255:-1:0]/255;         % thresholding image from 0 - 255
% [datasetStruct.GTdir '*.png']
GTfiles=dir([datasetStruct.GTdir '*.png']); % gather the GT files in png
GTfiles = [GTfiles; dir([datasetStruct.GTdir '*.jpg'])]; % gather the GT files in jpg
GTfiles = [GTfiles; dir([datasetStruct.GTdir '*.bmp'])]; % gather the GT files in bmp
GTfiles = [GTfiles; dir([datasetStruct.GTdir '*.tif'])]; % gather the GT files in tif

numOfFiles = size(GTfiles,1); % totalImgNum*1
numOfAlgos = length(algStructArray);  % num of algorithmos to be evaluated

%% initial

[TPs, FPs] = deal(zeros(length(thresholds), 1, numOfAlgos));
gtPxlNums = deal(zeros(numOfAlgos));

%Iterate over images
hwait  = waitbar(0,['Processing ', num2str(numOfFiles),' images on ',datasetStruct.datasetName,': 0%']);
% totalNum = numOfFiles* ones(numOfAlgos,1); %3*1   ( [totalImgNum;totalImgNum;totalImgNum;totalImgNum] ) 
    
for imIndx=1:numOfFiles  
    % read the ground truth files
    [~,base_name,ext] = fileparts(GTfiles(imIndx).name); 
    
    if strcmp(ext,'tif')
        gtMap = im2double(~imread([datasetStruct.GTdir base_name ext]));
    else
        gtMap = im2double(imread([datasetStruct.GTdir base_name ext]));
    end
    
    gtSize = size(gtMap); 
    if (length(gtSize) == 3)
        convert = 'rgb2gray'
        gtMap = rgb2gray(gtMap);
        gtSize(3)= [];
    end
    
    gtMap = logical(gtMap>=0.1); % get binary mask
    for algIdx = 1:numOfAlgos % compute over each algorithms iteratively
        if strcmp(ext,'tif')
            sMap = readSaliencyMap(algStructArray{algIdx}, ['pred_', base_name], gtSize, 'tif');% read and resize saliency maps
        else
            sMap = readSaliencyMap(algStructArray{algIdx}, [base_name], gtSize);% read and resize saliency maps
        end
        
        % count and pdf the TPs, FPs and gtPxlNums = TPs + FNs
        [gtPxl, TP, FP] = compute_TP_FP(sMap,thresholds,gtMap);
        TPs(:, :, algIdx) = TPs(:, :, algIdx) + TP;
        FPs(:, :, algIdx) = FPs(:, :, algIdx) + FP;
        gtPxlNums(algIdx) = gtPxlNums(algIdx) + gtPxl;

    end
    pct     = sprintf('%5.1f',imIndx/numOfFiles*100);
    waitbar(imIndx/numOfFiles,hwait,['Processing ', num2str(numOfFiles),...
        ' images on ',datasetStruct.datasetName,': ',pct,'%']); 
end %End of image loop

close(hwait);

[Pre, Recall] = deal(zeros(length(thresholds), numOfAlgos));
for algoIdx = 1:numOfAlgos

    targetHistc = cumsum(TPs(:, :, algoIdx));
    nontargetHistc = cumsum(FPs(:, :, algoIdx));
    pre = targetHistc ./ (targetHistc + nontargetHistc + 1e-8);
    recall = targetHistc ./ (gtPxlNums(algoIdx) + 1e-8); 
    Pre(:, algoIdx) = pre;
    Recall(:, algoIdx) = recall;
end

Metrics.Pre = Pre;
Metrics.Recall = Recall;

end

%% Read Saliency Maps

% Read and resize saliency map
function sMap = readSaliencyMap(algStruct,base_name,gtSize,ext)
if nargin < 4
    ext = 'png';
end

algStruct.ext=ext;
if strcmp(algStruct.dir(end-3:end),'DSR')
    algStruct.ext='jpg';
elseif strcmp(algStruct.dir(end-12:end-10),'DSR')
    algStruct.ext='jpg';
end

file_name = fullfile(algStruct.dir,[algStruct.prefix base_name algStruct.postfix '.' algStruct.ext]);
sMap = imresize(im2double(imread(file_name)),gtSize(1:2));
if (size(sMap,3)==3)
    sMap = rgb2gray(sMap);
end
sMap(sMap<0)=0;
maxnum = max(sMap(:));
if maxnum==0
    sMap = zeros(gtSize(1:2));
else
    sMap = sMap./maxnum;
end  

end

%% Calculate the scores
function [gtPxlNum, targetHist, nontargetHist] ...
    = thresholdBased_TP_FP(sMap,thresholds,gtMap) %calculate the threshold based scores
    gtPxlNum = sum(gtMap(:));
    
    % non-dereasing vectors, so reverse the thresholds
    targetHist = histc(sMap(gtMap), fliplr(thresholds));
    nontargetHist = histc(sMap(~gtMap), fliplr(thresholds));

    targetHist = flipud(targetHist);
    nontargetHist = flipud(nontargetHist);

    fnontargetHist = nontargetHist;

end