Benchmarking Cell Tracking Challenge Data

import os
from pathlib import Path
import pprint
import urllib.request
import zipfile

from tqdm import tqdm

from traccuracy import run_metrics
from traccuracy.loaders import load_ctc_data
from traccuracy.matchers import CTCMatcher, IOUMatcher
from traccuracy.metrics import CTCMetrics, DivisionMetrics

pp = pprint.PrettyPrinter(indent=4)

url = "http://data.celltrackingchallenge.net/training-datasets/Fluo-N2DL-HeLa.zip"
data_dir = 'downloads'

if not os.path.exists(data_dir):
    os.mkdir(data_dir)

filename = url.split('/')[-1]
file_path = os.path.join(data_dir, filename)
ds_name = filename.split('.')[0]

# Add a utility to make a progress bar when downloading the file
class DownloadProgressBar(tqdm):
    def update_to(self, b=1, bsize=1, tsize=None):
        if tsize is not None:
            self.total = tsize
        self.update(b * bsize - self.n)

if not os.path.exists(file_path):
    print(f"Downloading {ds_name} data from the CTC website")
    # Downloading data
    with DownloadProgressBar(unit='B', unit_scale=True,
                             miniters=1, desc=url.split('/')[-1]) as t:
        urllib.request.urlretrieve(url, file_path, reporthook=t.update_to)
    # Unzip the data
    # TODO add a progress bar to zip as well
    with zipfile.ZipFile(file_path, 'r') as zip_ref:
        zip_ref.extractall(data_dir)

gt_data = load_ctc_data(
    'downloads/Fluo-N2DL-HeLa/01_GT/TRA',
    'downloads/Fluo-N2DL-HeLa/01_GT/TRA/man_track.txt',
    name='Hela-01_GT'
)
pred_data = load_ctc_data(
    'sample-data/Fluo-N2DL-HeLa/01_RES',
    'sample-data/Fluo-N2DL-HeLa/01_RES/res_track.txt',
    name='Hela-01_RES'
)

Loading TIFFs:   0%|          | 0/92 [00:00<?, ?it/s]

Loading TIFFs: 100%|██████████| 92/92 [00:00<00:00, 402.71it/s]
Loading TIFFs: 100%|██████████| 92/92 [00:00<00:00, 609.62it/s]

Run CTC metrics with additional evaluation of division events.

ctc_results = run_metrics(
    gt_data=gt_data,
    pred_data=pred_data,
    matcher=CTCMatcher(),
    metrics=[CTCMetrics(), DivisionMetrics(frame_buffer=(0,1,2))],
)
pp.pprint(ctc_results)

Matching frames: 100%|██████████| 92/92 [00:01<00:00, 90.00it/s]
Evaluating nodes: 100%|██████████| 8600/8600 [00:00<00:00, 645392.99it/s]
Evaluating FP edges: 100%|██████████| 8535/8535 [00:00<00:00, 923996.20it/s]
Evaluating FN edges: 100%|██████████| 8562/8562 [00:00<00:00, 990392.47it/s]

[   {   'gt': 'Hela-01_GT',
        'matcher': {'name': 'CTCMatcher'},
        'metric': {   'e_weights': {'fn': 1.5, 'fp': 1, 'ws': 1},
                      'name': 'CTCMetrics',
                      'v_weights': {'fn': 10, 'fp': 1, 'ns': 5}},
        'pred': 'Hela-01_RES',
        'results': {   'AOGM': 627.5,
                       'DET': 0.9954855886097927,
                       'TRA': 0.993676498745377,
                       'fn_edges': 87,
                       'fn_nodes': 39,
                       'fp_edges': 60,
                       'fp_nodes': 0,
                       'ns_nodes': 0,
                       'ws_edges': 47},
        'version': '0.0.3.dev5+gf8c982c'},
    {   'gt': 'Hela-01_GT',
        'matcher': {'name': 'CTCMatcher'},
        'metric': {'frame_buffer': (0, 1, 2), 'name': 'DivisionMetrics'},
        'pred': 'Hela-01_RES',
        'results': {   'Frame Buffer 0': {   'Division F1': 0.76,
                                             'Division Precision': 0.7169811320754716,
                                             'Division Recall': 0.8085106382978723,
                                             'False Negative Divisions': 18,
                                             'False Positive Divisions': 30,
                                             'Mitotic Branching Correctness': 0.6129032258064516,
                                             'True Positive Divisions': 76},
                       'Frame Buffer 1': {   'Division F1': 0.76,
                                             'Division Precision': 0.7169811320754716,
                                             'Division Recall': 0.8085106382978723,
                                             'False Negative Divisions': 18,
                                             'False Positive Divisions': 30,
                                             'Mitotic Branching Correctness': 0.6129032258064516,
                                             'True Positive Divisions': 76},
                       'Frame Buffer 2': {   'Division F1': 0.76,
                                             'Division Precision': 0.7169811320754716,
                                             'Division Recall': 0.8085106382978723,
                                             'False Negative Divisions': 18,
                                             'False Positive Divisions': 30,
                                             'Mitotic Branching Correctness': 0.6129032258064516,
                                             'True Positive Divisions': 76}},
        'version': '0.0.3.dev5+gf8c982c'}]

Use an IOU matcher which supports a minimum threshold for overlap and run division metrics.

iou_results = run_metrics(
    gt_data=gt_data,
    pred_data=pred_data,
    matcher=IOUMatcher(iou_threshold=0.1),
    metrics=[DivisionMetrics(frame_buffer=(0,1,2))],
)
pp.pprint(iou_results)

Matching frames: 100%|██████████| 92/92 [00:15<00:00,  6.01it/s]

[   {   'gt': 'Hela-01_GT',
        'matcher': {'iou_threshold': 0.1, 'name': 'IOUMatcher'},
        'metric': {'frame_buffer': (0, 1, 2), 'name': 'DivisionMetrics'},
        'pred': 'Hela-01_RES',
        'results': {   'Frame Buffer 0': {   'Division F1': 0.711340206185567,
                                             'Division Precision': 0.69,
                                             'Division Recall': 0.7340425531914894,
                                             'False Negative Divisions': 25,
                                             'False Positive Divisions': 31,
                                             'Mitotic Branching Correctness': 0.552,
                                             'True Positive Divisions': 69},
                       'Frame Buffer 1': {   'Division F1': 0.711340206185567,
                                             'Division Precision': 0.69,
                                             'Division Recall': 0.7340425531914894,
                                             'False Negative Divisions': 25,
                                             'False Positive Divisions': 31,
                                             'Mitotic Branching Correctness': 0.552,
                                             'True Positive Divisions': 69},
                       'Frame Buffer 2': {   'Division F1': 0.711340206185567,
                                             'Division Precision': 0.69,
                                             'Division Recall': 0.7340425531914894,
                                             'False Negative Divisions': 25,
                                             'False Positive Divisions': 31,
                                             'Mitotic Branching Correctness': 0.552,
                                             'True Positive Divisions': 69}},
        'version': '0.0.3.dev5+gf8c982c'}]