Source code for cw_eval.challenge_eval.off_nadir_dataset

from __future__ import print_function, with_statement, division

import pandas as pd
from cw_eval import baseeval as bF
import re

# Note, for mac osx compatability import something from shapely.geometry before
# importing fiona or geopandas
#  * Import shapely before
# rasterio or fiona

[docs]def eval_off_nadir(prop_csv, truth_csv, imageColumns={}, miniou=0.5, minArea=20): """Evaluate an off-nadir competition proposal csv. Uses :class:`EvalBase` to evaluate off-nadir challenge proposals. See ``imageColumns`` in the source code for how collects are broken into Nadir, Off-Nadir, and Very-Off-Nadir bins. Arguments --------- prop_csv : str Path to the proposal polygon CSV file. truth_csv : str Path to the ground truth polygon CSV file. imageColumns : dict, optional dict of ``(collect: nadir bin)`` pairs used to separate collects into sets. Nadir bin values must be one of ``["Nadir", "Off-Nadir", "Very-Off-Nadir"]`` . See source code for collect name options. miniou : float, optional Minimum IoU score between a region proposal and ground truth to define as a successful identification. Defaults to 0.5. minArea : float or int, optional Minimum area of ground truth regions to include in scoring calculation. Defaults to ``20``. Returnss ------- results_DF, results_DF_Full results_DF : :py:class:`pd.DataFrame` Summary :py:class:`pd.DataFrame` of score outputs grouped by nadir angle bin, along with the overall score. results_DF_Full : :py:class:`pd.DataFrame` :py:class:`pd.DataFrame` of scores by individual image chip across the ground truth and proposal datasets. """ evalObject = bF.EvalBase(ground_truth_vector_file=truth_csv) evalObject.load_proposal(prop_csv, conf_field_list=['Confidence'], proposalCSV=True ) results = evalObject.eval_iou_spacenet_csv(miniou=miniou, iou_field_prefix="iou_score", imageIDField="ImageId", minArea=minArea ) results_DF_Full = pd.DataFrame(results) if not imageColumns: imageColumns = { 'Atlanta_nadir7_catid_1030010003D22F00': "Nadir", 'Atlanta_nadir8_catid_10300100023BC100': "Nadir", 'Atlanta_nadir10_catid_1030010003993E00': "Nadir", 'Atlanta_nadir10_catid_1030010003CAF100': "Nadir", 'Atlanta_nadir13_catid_1030010002B7D800': "Nadir", 'Atlanta_nadir14_catid_10300100039AB000': "Nadir", 'Atlanta_nadir16_catid_1030010002649200': "Nadir", 'Atlanta_nadir19_catid_1030010003C92000': "Nadir", 'Atlanta_nadir21_catid_1030010003127500': "Nadir", 'Atlanta_nadir23_catid_103001000352C200': "Nadir", 'Atlanta_nadir25_catid_103001000307D800': "Nadir", 'Atlanta_nadir27_catid_1030010003472200': "Off-Nadir", 'Atlanta_nadir29_catid_1030010003315300': "Off-Nadir", 'Atlanta_nadir30_catid_10300100036D5200': "Off-Nadir", 'Atlanta_nadir32_catid_103001000392F600': "Off-Nadir", 'Atlanta_nadir34_catid_1030010003697400': "Off-Nadir", 'Atlanta_nadir36_catid_1030010003895500': "Off-Nadir", 'Atlanta_nadir39_catid_1030010003832800': "Off-Nadir", 'Atlanta_nadir42_catid_10300100035D1B00': "Very-Off-Nadir", 'Atlanta_nadir44_catid_1030010003CCD700': "Very-Off-Nadir", 'Atlanta_nadir46_catid_1030010003713C00': "Very-Off-Nadir", 'Atlanta_nadir47_catid_10300100033C5200': "Very-Off-Nadir", 'Atlanta_nadir49_catid_1030010003492700': "Very-Off-Nadir", 'Atlanta_nadir50_catid_10300100039E6200': "Very-Off-Nadir", 'Atlanta_nadir52_catid_1030010003BDDC00': "Very-Off-Nadir", 'Atlanta_nadir53_catid_1030010003193D00': "Very-Off-Nadir", 'Atlanta_nadir53_catid_1030010003CD4300': "Very-Off-Nadir", } results_DF_Full['nadir-category'] = [ imageColumns[get_collect_id(imageID)] for imageID in results_DF_Full['imageID'].values] results_DF = results_DF_Full.groupby(['nadir-category']).sum() # Recalculate Values after Summation of AOIs for indexVal in results_DF.index: rowValue = results_DF[results_DF.index == indexVal] # Precision = TruePos / float(TruePos + FalsePos) if float(rowValue['TruePos'] + rowValue['FalsePos']) > 0: Precision = float( rowValue['TruePos'] / float(rowValue['TruePos'] + rowValue['FalsePos']) ) else: Precision = 0 # Recall = TruePos / float(TruePos + FalseNeg) if float(rowValue['TruePos'] + rowValue['FalseNeg']) > 0: Recall = float(rowValue['TruePos'] / float(rowValue['TruePos'] + rowValue['FalseNeg'])) else: Recall = 0 if Recall * Precision > 0: F1Score = 2 * Precision * Recall / (Precision + Recall) else: F1Score = 0 results_DF.loc[results_DF.index == indexVal, 'Precision'] = Precision results_DF.loc[results_DF.index == indexVal, 'Recall'] = Recall results_DF.loc[results_DF.index == indexVal, 'F1Score'] = F1Score return results_DF, results_DF_Full
[docs]def get_collect_id(imageID): """Get the collect ID for an image name using a regex.""" collect_id = re.findall('Atlanta_nadir[0-9]{1,2}_catid_[0-9A-Z]{16}', imageID)[0] return collect_id