Source code for solaris.eval.pixel

import time
import cv2
import numpy as np
import matplotlib
import matplotlib.pyplot as plt


[docs]def iou(truth_mask, prop_mask, prop_threshold=0.5, verbose=False): """Compute pixel-wise intersection over union. Multiplies truth_mask by 2, and subtract. Make sure arrays are clipped so that overlapping regions don't cause problems Arguments --------- truth_mask : :class:`numpy.ndarray` 2-D binary array of ground truth pixels. prop_mask : :class:`numpy.ndarray` 2-D array of proposals. prop_threshold : float, optional The threshold for proposal values to be defined as positive (``1``) or negative (``0``) predictions. Values >= `prop_threshold` will be set to ``1``, values < `prop_threshold` will be set to ``0``. verbose : bool, optional Switch to print relevant values. Returns ------- iou : float Intersection over union of ground truth and proposal """ if truth_mask.shape != prop_mask.shape: raise ValueError("The shape of `truth_mask` and `prop_mask` must " "be the same.") truth_mask_clip = np.clip(truth_mask, 0, 1).astype(float) prop_mask_clip = (np.clip(prop_mask, 0, 1) >= prop_threshold).astype(float) # subtract array sub_mask = 2*prop_mask_clip - truth_mask_clip add_mask = prop_mask_clip + truth_mask_clip # true pos = 1, false_pos = 2, true_neg = 0, false_neg = -1 tp_count = np.sum(sub_mask == 1) union = np.sum(add_mask > 0) intersection = tp_count iou = 1. * intersection / union if verbose: print("intersection:", intersection) print("union:", union) print("iou:", iou) return iou
[docs]def f1(truth_mask, prop_mask, prop_threshold=0.5, show_plot=False, im_file='', show_colorbar=False, plot_file='', dpi=200, verbose=False): """Compute pixel-wise precision, recall, and f1 score. Find true pos, false pos, true neg, false neg as well as f1 score. Multiply truth_mask by 2, and subtract. Make sure arrays are clipped so that overlapping regions don't cause problems. Arguments --------- truth_mask : :class:`numpy.ndarray` 2-D binary array of ground truth pixels. prop_mask : :class:`numpy.ndarray` 2-D array of proposals. prop_threshold : float, optional The threshold for proposal values to be defined as positive (``1``) or negative (``0``) predictions. Values >= `prop_threshold` will be set to ``1``, values < `prop_threshold` will be set to ``0``. show_plot : bool, optional Switch to plot the outputs. Defaults to ``False``. im_file : str, optional Image file corresponding to the masks. Ignored if ``show_plot == False``. Defaults to ``''``. show_colorbar : bool, optional Switch to show colorbar. Ignored if ``show_plot == False``. Defaults to ``False``. plot_file : str, optional Output file if plotting. Ignored if ``show_plot == False``. Defaults to ``''``. dpi : int, optional Dots per inch for plotting. Ignored if ``show_plot == False``. Defaults to ``200``. verbose : bool, optional Switch to print relevant values. Returns ------- f1 : float Pixel-wise F1 score. precision : float Pixel-wise precision. recall : float Pixel-wise recall. """ truth_mask_clip = np.clip(truth_mask, 0, 1).astype(float) prop_mask_clip = (np.clip(prop_mask, 0, 1) >= prop_threshold).astype(float) # subtract array sub_mask = 2*prop_mask_clip - truth_mask_clip # sub_mask2 = prop_mask_clip - truth_mask_clip # true pos = 1, false_pos = 2, true_neg = 0, false_neg = -1 n_pos = len(np.where(truth_mask_clip == 1)[0]) tp_count = len(np.where(sub_mask == 1)[0]) fp_count = len(np.where(sub_mask == 2)[0]) tn_count = len(np.where(sub_mask == 0)[0]) fn_count = len(np.where(sub_mask == -1)[0]) if (n_pos > 0) and (tp_count > 0): precision = float(tp_count) / float(tp_count + fp_count) recall = float(tp_count) / float(tp_count + fn_count) f1 = 2 * precision * recall / (precision + recall) else: precision, recall, f1 = 0, 0, 0 if verbose: print("mask.shape:\t", truth_mask.shape) print("num pixels:\t", truth_mask.size) print("false_neg:\t", fn_count) print("false_pos:\t", fp_count) print("true_neg:\t", tn_count) print("true_pos:\t", tp_count) print("precision:\t", precision) print("recall:\t\t", recall) print("F1 score:\t", f1) # TODO: split this out into a separate function if show_plot: fontsize = 6 t0 = time.time() title = "Precision: " + str(np.round(precision, 3)) \ + " Recall: " + str(np.round(recall, 3)) \ + " F1: " + str(np.round(f1, 3)) if show_colorbar: fig, ((ax1, ax2), (ax3, ax4)) = plt.subplots(2, 2, sharex=True, sharey=True, figsize=(6, 6)) else: fig, (ax1, ax2, ax3) = plt.subplots(1, 3, sharex=True, sharey=True, figsize=(9.5, 3)) # fig, ((ax1, ax2, ax3)) = plt.subplots(1, 3, sharex=True, # sharey=True, figsize=(13,4)) plt.suptitle(title, fontsize=fontsize) # ground truth if len(im_file) > 0: # raw image ax1.imshow(cv2.imread(im_file, 1)) # ground truth # set zeros to nan palette = plt.cm.gray palette.set_over('orange', 1.0) palette.set_over('orange', 1.0) z = truth_mask.astype(float) z[z == 0] = np.nan ax1.imshow(z, cmap=palette, alpha=0.5, norm=matplotlib.colors.Normalize( vmin=0.5, vmax=0.9, clip=False)) ax1.set_title('truth_mask_clip + image', fontsize=fontsize) else: ax1.imshow(truth_mask_clip) ax1.set_title('truth_mask_clip', fontsize=fontsize) ax1.axis('off') # proposal mask ax2.imshow(prop_mask_clip) ax2.axis('off') ax2.set_title('prop_mask_clip', fontsize=fontsize) # mask if show_colorbar: z = ax3.pcolor(sub_mask) fig.colorbar(z) ax4.axis('off') else: ax3.imshow(sub_mask) # z = ax3.pcolor(sub_mask2) ax3.axis('off') ax3.set_title('subtract_mask', fontsize=fontsize) # plt.tight_layout() # fig.tight_layout(rect=[0, 0.03, 1, 0.95]) plt.subplots_adjust(top=0.8) if len(plot_file) > 0: plt.savefig(plot_file, dpi=dpi) print("Time to create and save F1 plots:", time.time() - t0, "seconds") plt.show() return f1, precision, recall
def _get_neighborhood_limits(row, col, h, w, rho=3): '''Get neighbors of point p with pixel coords row, col''' rowmin = max(0, row-rho) rowmax = min(h, row + rho) colmin = max(0, col-rho) colmax = min(w, col + rho) return rowmin, rowmax, colmin, colmax
[docs]def relaxed_f1(truth_mask, prop_mask, radius=3, verbose=False): """ Compute relaxed f1 score Notes ----- Also find relaxed precision, recall, f1. http://www.cs.toronto.edu/~fritz/absps/road_detection.pdf "completenetess represents the fraction of true road pixels that are within ρ pixels of a predicted road pixel, while correctness measures the fraction of predicted road pixels that are within ρ pixels of a true road pixel." https://arxiv.org/pdf/1711.10684.pdf The relaxed precision is defined as the fraction of number of pixels predicted as road within a range of ρ pixels from pixels labeled as road. The relaxed recall is the fraction of number of pixels labeled as road that are within a range of ρ pixels from pixels predicted as road. http://ceur-ws.org/Vol-156/paper5.pdf Arguments --------- truth_mask : np array 2-D array of ground truth. prop_mask : np array 2-D array of proposals. radius : int Radius in pixels to use for relaxed f1. verbose : bool Switch to print relevant values Returns ------- output : tuple Tuple containing [relaxed_f1, relaxed_precision, relaxed_recall] Examples -------- >>> truth_mask = np.zeros(shape=(10, 10)) >>> prop_mask = np.zeros(shape=(10, 10)) >>> truth_mask[5, :] = 1 >>> prop_mask[5, :] = 1 >>> prop_mask[:, 2] = 0 >>> prop_mask[:, 3] = 1 >>> prop_mask[6:8, :] = 0 >>> prop_mask array([[0., 0., 0., 1., 0., 0., 0., 0., 0., 0.], [0., 0., 0., 1., 0., 0., 0., 0., 0., 0.], [0., 0., 0., 1., 0., 0., 0., 0., 0., 0.], [0., 0., 0., 1., 0., 0., 0., 0., 0., 0.], [0., 0., 0., 1., 0., 0., 0., 0., 0., 0.], [1., 1., 0., 1., 1., 1., 1., 1., 1., 1.], [0., 0., 0., 0., 0., 0., 0., 0., 0., 0.], [0., 0., 0., 0., 0., 0., 0., 0., 0., 0.], [0., 0., 0., 1., 0., 0., 0., 0., 0., 0.], [0., 0., 0., 1., 0., 0., 0., 0., 0., 0.]]) >>>truth_mask array([[0., 0., 0., 0., 0., 0., 0., 0., 0., 0.], [0., 0., 0., 0., 0., 0., 0., 0., 0., 0.], [0., 0., 0., 0., 0., 0., 0., 0., 0., 0.], [0., 0., 0., 0., 0., 0., 0., 0., 0., 0.], [0., 0., 0., 0., 0., 0., 0., 0., 0., 0.], [1., 1., 1., 1., 1., 1., 1., 1., 1., 1.], [0., 0., 0., 0., 0., 0., 0., 0., 0., 0.], [0., 0., 0., 0., 0., 0., 0., 0., 0., 0.], [0., 0., 0., 0., 0., 0., 0., 0., 0., 0.], [0., 0., 0., 0., 0., 0., 0., 0., 0., 0.]]) >>> relaxed_f1(truth_mask, prop_mask, radius=3) (0.8571428571428571, 0.75, 1.0) """ truth_mask_clip = np.clip(truth_mask, 0, 1).astype(float) prop_mask_clip = np.clip(prop_mask, 0, 1).astype(float) # true pos = 1, false_pos = 2, true_neg = 0, false_neg = -1 n_truth = len(np.where(truth_mask_clip == 1)[0]) n_prop = len(np.where(prop_mask_clip == 1)[0]) # iterate through truth pixels precision_count = 0 recall_count = 0 h, w = truth_mask.shape for row in range(h): for col in range(w): truth_val = truth_mask_clip[row][col] prop_val = prop_mask_clip[row][col] # get window limits rowmin, rowmax, colmin, colmax = _get_neighborhood_limits( row, col, h, w, rho=radius) # get windows truth_win = truth_mask_clip[rowmin:rowmax, colmin:colmax] prop_win = prop_mask_clip[rowmin:rowmax, colmin:colmax] # add precision_count if proposal is within the radius of a gt node if prop_val == 1: if np.max(truth_win) > 0: precision_count += 1 if truth_val == 1: if np.max(prop_win) > 0: recall_count += 1 # get fractions if n_truth == 0: relaxed_recall = 0 else: relaxed_recall = 1. * recall_count / n_truth if n_prop == 0: relaxed_precision = 0 else: relaxed_precision = 1. * precision_count / n_prop if (relaxed_recall > 0) and (relaxed_precision > 0): relaxed_f1 = 2 * relaxed_precision * relaxed_recall \ / (relaxed_precision + relaxed_recall) else: relaxed_f1 = 0 if verbose: print("mask.shape:\t", truth_mask.shape) print("num pixels:\t", truth_mask.size) print("precision:\t", relaxed_precision) print("recall:\t\t", relaxed_recall) print("rF1 score:\t", f1) output = (relaxed_f1, relaxed_precision, relaxed_recall) return output