eval_coco_format.py

# Lint as: python2, python3
# Copyright 2019 The TensorFlow Authors All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Computes evaluation metrics on groundtruth and predictions in COCO format.

The Common Objects in Context (COCO) dataset defines a format for specifying
combined semantic and instance segmentations as "panoptic" segmentations. This
is done with the combination of JSON and image files as specified at:
http://cocodataset.org/#format-results
where the JSON file specifies the overall structure of the result,
including the categories for each annotation, and the images specify the image
region for each annotation in that image by its ID.

This script computes additional metrics such as Parsing Covering on datasets and
predictions in this format. An implementation of Panoptic Quality is also
provided for convenience.
"""

from __future__ importabsolute_import
from __future__ importdivision
from __future__ importprint_function

importcollections
importjson
importmultiprocessing
importos

fromabslimportapp
fromabslimportflags
fromabslimportlogging
importnumpyasnp
fromPILimportImage
importutilsaspanopticapi_utils
importsix

fromdeeplab.evaluationimportpanoptic_quality
fromdeeplab.evaluationimportparsing_covering

FLAGS=flags.FLAGS

flags.DEFINE_string(
'gt_json_file', None,
' Path to a JSON file giving ground-truth annotations in COCO format.')
flags.DEFINE_string('pred_json_file', None,
'Path to a JSON file for the predictions to evaluate.')
flags.DEFINE_string(
'gt_folder', None,
'Folder containing panoptic-format ID images to match ground-truth '
'annotations to image regions.')
flags.DEFINE_string('pred_folder', None,
'Folder containing ID images for predictions.')
flags.DEFINE_enum(
'metric', 'pq', ['pq', 'pc'], 'Shorthand name of a metric to compute. '
'Supported values are:\n'
'Panoptic Quality (pq)\n'
'Parsing Covering (pc)')
flags.DEFINE_integer(
'num_categories', 201,
'The number of segmentation categories (or "classes") in the dataset.')
flags.DEFINE_integer(
'ignored_label', 0,
'A category id that is ignored in evaluation, e.g. the void label as '
'defined in COCO panoptic segmentation dataset.')
flags.DEFINE_integer(
'max_instances_per_category', 256,
'The maximum number of instances for each category. Used in ensuring '
'unique instance labels.')
flags.DEFINE_integer('intersection_offset', None,
'The maximum number of unique labels.')
flags.DEFINE_bool(
'normalize_by_image_size', True,
'Whether to normalize groundtruth instance region areas by image size. If '
'True, groundtruth instance areas and weighted IoUs will be divided by the '
'size of the corresponding image before accumulated across the dataset. '
'Only used for Parsing Covering (pc) evaluation.')
flags.DEFINE_integer(
'num_workers', 0, 'If set to a positive number, will spawn child processes '
'to compute parts of the metric in parallel by splitting '
'the images between the workers. If set to -1, will use '
'the value of multiprocessing.cpu_count().')
flags.DEFINE_integer('print_digits', 3,
'Number of significant digits to print in metrics.')


def_build_metric(metric,
num_categories,
ignored_label,
max_instances_per_category,
intersection_offset=None,
normalize_by_image_size=True):
"""Creates a metric aggregator objet of the given name."""
ifmetric=='pq':
logging.warning('One should check Panoptic Quality results against the '
'official COCO API code. Small numerical differences '
'(< 0.1%) can be magnified by rounding.')
returnpanoptic_quality.PanopticQuality(num_categories, ignored_label,
max_instances_per_category,
intersection_offset)
elifmetric=='pc':
returnparsing_covering.ParsingCovering(
num_categories, ignored_label, max_instances_per_category,
intersection_offset, normalize_by_image_size)
else:
raiseValueError('No implementation for metric "%s"'%metric)


def_matched_annotations(gt_json, pred_json):
"""Yields a set of (groundtruth, prediction) image annotation pairs.."""
image_id_to_pred_ann= {
annotation['image_id']: annotation
forannotationinpred_json['annotations']
 }
forgt_anningt_json['annotations']:
image_id=gt_ann['image_id']
pred_ann=image_id_to_pred_ann[image_id]
yieldgt_ann, pred_ann


def_open_panoptic_id_image(image_path):
"""Loads a COCO-format panoptic ID image from file."""
returnpanopticapi_utils.rgb2id(
np.array(Image.open(image_path), dtype=np.uint32))


def_split_panoptic(ann_json, id_array, ignored_label, allow_crowds):
"""Given the COCO JSON and ID map, splits into categories and instances."""
category=np.zeros(id_array.shape, np.uint16)
instance=np.zeros(id_array.shape, np.uint16)
next_instance_id=collections.defaultdict(int)
# Skip instance label 0 for ignored label. That is reserved for void.
next_instance_id[ignored_label] =1
forsegment_infoinann_json['segments_info']:
ifallow_crowdsandsegment_info['iscrowd']:
category_id=ignored_label
else:
category_id=segment_info['category_id']
mask=np.equal(id_array, segment_info['id'])
category[mask] =category_id
instance[mask] =next_instance_id[category_id]
next_instance_id[category_id] +=1
returncategory, instance


def_category_and_instance_from_annotation(ann_json, folder, ignored_label,
allow_crowds):
"""Given the COCO JSON annotations, finds maps of categories and instances."""
panoptic_id_image=_open_panoptic_id_image(
os.path.join(folder, ann_json['file_name']))
return_split_panoptic(ann_json, panoptic_id_image, ignored_label,
allow_crowds)


def_compute_metric(metric_aggregator, gt_folder, pred_folder,
annotation_pairs):
"""Iterates over matched annotation pairs and computes a metric over them."""
forgt_ann, pred_anninannotation_pairs:
# We only expect "iscrowd" to appear in the ground-truth, and not in model
# output. In predicted JSON it is simply ignored, as done in official code.
gt_category, gt_instance=_category_and_instance_from_annotation(
gt_ann, gt_folder, metric_aggregator.ignored_label, True)
pred_category, pred_instance=_category_and_instance_from_annotation(
pred_ann, pred_folder, metric_aggregator.ignored_label, False)

metric_aggregator.compare_and_accumulate(gt_category, gt_instance,
pred_category, pred_instance)
returnmetric_aggregator


def_iterate_work_queue(work_queue):
"""Creates an iterable that retrieves items from a queue until one is None."""
task=work_queue.get(block=True)
whiletaskisnotNone:
yieldtask
task=work_queue.get(block=True)


def_run_metrics_worker(metric_aggregator, gt_folder, pred_folder, work_queue,
result_queue):
result=_compute_metric(metric_aggregator, gt_folder, pred_folder,
_iterate_work_queue(work_queue))
result_queue.put(result, block=True)


def_is_thing_array(categories_json, ignored_label):
"""is_thing[category_id] is a bool on if category is "thing" or "stuff"."""
is_thing_dict= {}
forcategory_jsonincategories_json:
is_thing_dict[category_json['id']] =bool(category_json['isthing'])

# Check our assumption that the category ids are consecutive.
# Usually metrics should be able to handle this case, but adding a warning
# here.
max_category_id=max(six.iterkeys(is_thing_dict))
iflen(is_thing_dict) !=max_category_id+1:
seen_ids=six.viewkeys(is_thing_dict)
all_ids=set(six.moves.range(max_category_id+1))
unseen_ids=all_ids.difference(seen_ids)
ifunseen_ids!= {ignored_label}:
logging.warning(
'Nonconsecutive category ids or no category JSON specified for ids: '
'%s', unseen_ids)

is_thing_array=np.zeros(max_category_id+1)
forcategory_id, is_thinginsix.iteritems(is_thing_dict):
is_thing_array[category_id] =is_thing

returnis_thing_array


defeval_coco_format(gt_json_file,
pred_json_file,
gt_folder=None,
pred_folder=None,
metric='pq',
num_categories=201,
ignored_label=0,
max_instances_per_category=256,
intersection_offset=None,
normalize_by_image_size=True,
num_workers=0,
print_digits=3):
"""Top-level code to compute metrics on a COCO-format result.

 Note that the default values are set for COCO panoptic segmentation dataset,
 and thus the users may want to change it for their own dataset evaluation.

 Args:
 gt_json_file: Path to a JSON file giving ground-truth annotations in COCO
 format.
 pred_json_file: Path to a JSON file for the predictions to evaluate.
 gt_folder: Folder containing panoptic-format ID images to match ground-truth
 annotations to image regions.
 pred_folder: Folder containing ID images for predictions.
 metric: Name of a metric to compute.
 num_categories: The number of segmentation categories (or "classes") in the
 dataset.
 ignored_label: A category id that is ignored in evaluation, e.g. the "void"
 label as defined in the COCO panoptic segmentation dataset.
 max_instances_per_category: The maximum number of instances for each
 category. Used in ensuring unique instance labels.
 intersection_offset: The maximum number of unique labels.
 normalize_by_image_size: Whether to normalize groundtruth instance region
 areas by image size. If True, groundtruth instance areas and weighted IoUs
 will be divided by the size of the corresponding image before accumulated
 across the dataset. Only used for Parsing Covering (pc) evaluation.
 num_workers: If set to a positive number, will spawn child processes to
 compute parts of the metric in parallel by splitting the images between
 the workers. If set to -1, will use the value of
 multiprocessing.cpu_count().
 print_digits: Number of significant digits to print in summary of computed
 metrics.

 Returns:
 The computed result of the metric as a float scalar.
 """
withopen(gt_json_file, 'r') asgt_json_fo:
gt_json=json.load(gt_json_fo)
withopen(pred_json_file, 'r') aspred_json_fo:
pred_json=json.load(pred_json_fo)
ifgt_folderisNone:
gt_folder=gt_json_file.replace('.json', '')
ifpred_folderisNone:
pred_folder=pred_json_file.replace('.json', '')
ifintersection_offsetisNone:
intersection_offset= (num_categories+1) *max_instances_per_category

metric_aggregator=_build_metric(
metric, num_categories, ignored_label, max_instances_per_category,
intersection_offset, normalize_by_image_size)

ifnum_workers==-1:
logging.info('Attempting to get the CPU count to set # workers.')
num_workers=multiprocessing.cpu_count()

ifnum_workers>0:
logging.info('Computing metric in parallel with %d workers.', num_workers)
work_queue=multiprocessing.Queue()
result_queue=multiprocessing.Queue()
workers= []
worker_args= (metric_aggregator, gt_folder, pred_folder, work_queue,
result_queue)
for_insix.moves.range(num_workers):
workers.append(
multiprocessing.Process(target=_run_metrics_worker, args=worker_args))
forworkerinworkers:
worker.start()
forann_pairin_matched_annotations(gt_json, pred_json):
work_queue.put(ann_pair, block=True)

# Will cause each worker to return a result and terminate upon recieving a
# None task.
for_insix.moves.range(num_workers):
work_queue.put(None, block=True)

# Retrieve results.
for_insix.moves.range(num_workers):
metric_aggregator.merge(result_queue.get(block=True))

forworkerinworkers:
worker.join()
else:
logging.info('Computing metric in a single process.')
annotation_pairs=_matched_annotations(gt_json, pred_json)
_compute_metric(metric_aggregator, gt_folder, pred_folder, annotation_pairs)

is_thing=_is_thing_array(gt_json['categories'], ignored_label)
metric_aggregator.print_detailed_results(
is_thing=is_thing, print_digits=print_digits)
returnmetric_aggregator.detailed_results(is_thing=is_thing)


defmain(argv):
iflen(argv) >1:
raiseapp.UsageError('Too many command-line arguments.')

eval_coco_format(FLAGS.gt_json_file, FLAGS.pred_json_file, FLAGS.gt_folder,
FLAGS.pred_folder, FLAGS.metric, FLAGS.num_categories,
FLAGS.ignored_label, FLAGS.max_instances_per_category,
FLAGS.intersection_offset, FLAGS.normalize_by_image_size,
FLAGS.num_workers, FLAGS.print_digits)


if__name__=='__main__':
flags.mark_flags_as_required(
 ['gt_json_file', 'gt_folder', 'pred_json_file', 'pred_folder'])
app.run(main)