Benchmarking Cell Tracking Challenge Data
[ ]:
import os
import pprint
import urllib.request
import zipfile
from tqdm import tqdm
from traccuracy import run_metrics
from traccuracy.loaders import load_ctc_data
from traccuracy.matchers import CTCMatcher, IOUMatcher
from traccuracy.metrics import CTCMetrics, DivisionMetrics
from traccuracy.utils import export_graphs_to_geff
pp = pprint.PrettyPrinter(indent=4)
[3]:
url = "http://data.celltrackingchallenge.net/training-datasets/Fluo-N2DL-HeLa.zip"
data_dir = "downloads"
if not os.path.exists(data_dir):
os.mkdir(data_dir)
filename = url.split("/")[-1]
file_path = os.path.join(data_dir, filename)
ds_name = filename.split(".")[0]
[4]:
# Add a utility to make a progress bar when downloading the file
class DownloadProgressBar(tqdm):
def update_to(self, b=1, bsize=1, tsize=None):
if tsize is not None:
self.total = tsize
self.update(b * bsize - self.n)
if not os.path.exists(file_path):
print(f"Downloading {ds_name} data from the CTC website")
# Downloading data
with DownloadProgressBar(unit="B", unit_scale=True, miniters=1, desc=url.split("/")[-1]) as t:
urllib.request.urlretrieve(url, file_path, reporthook=t.update_to)
# Unzip the data
# TODO add a progress bar to zip as well
with zipfile.ZipFile(file_path, "r") as zip_ref:
zip_ref.extractall(data_dir)
Downloading Fluo-N2DL-HeLa data from the CTC website
Fluo-N2DL-HeLa.zip: 199MB [00:18, 10.6MB/s]
[5]:
gt_data = load_ctc_data(
"downloads/Fluo-N2DL-HeLa/01_GT/TRA",
"downloads/Fluo-N2DL-HeLa/01_GT/TRA/man_track.txt",
name="Hela-01_GT",
)
pred_data = load_ctc_data(
"sample-data/Fluo-N2DL-HeLa/01_RES",
"sample-data/Fluo-N2DL-HeLa/01_RES/res_track.txt",
name="Hela-01_RES",
)
Loading TIFFs: 100%|██████████| 91/91 [00:00<00:00, 298.63it/s]
Computing node attributes: 100%|██████████| 92/92 [00:00<00:00, 201.73it/s]
1 non-connected masks at t=23.
2 non-connected masks at t=52.
Loading TIFFs: 100%|██████████| 91/91 [00:00<00:00, 306.09it/s]
Computing node attributes: 100%|██████████| 92/92 [00:00<00:00, 188.62it/s]
Run CTC metrics with additional evaluation of division events.
[6]:
ctc_results, ctc_matched = run_metrics(
gt_data=gt_data,
pred_data=pred_data,
matcher=CTCMatcher(),
metrics=[CTCMetrics()],
)
pp.pprint(ctc_results)
Matching frames: 100%|██████████| 92/92 [00:00<00:00, 647.08it/s]
Evaluating nodes: 100%|██████████| 8600/8600 [00:00<00:00, 343775.75it/s]
Evaluating FP edges: 100%|██████████| 8535/8535 [00:00<00:00, 340023.79it/s]
Evaluating FN edges: 100%|██████████| 8562/8562 [00:00<00:00, 432316.06it/s]
[ { 'gt': 'Hela-01_GT',
'matcher': {'matching type': 'one-to-one', 'name': 'CTCMatcher'},
'metric': { 'e_weights': {'fn': 1.5, 'fp': 1, 'ws': 1},
'name': 'CTCMetrics',
'relax_skips_gt': False,
'relax_skips_pred': False,
'v_weights': {'fn': 10, 'fp': 1, 'ns': 5},
'valid_match_types': ['one-to-one', 'many-to-one']},
'pred': 'Hela-01_RES',
'results': { 'AOGM': 627.5,
'DET': 0.9954855886097927,
'LNK': 0.9815074359573308,
'TRA': 0.993676498745377,
'fn_edges': 87,
'fn_nodes': 39,
'fp_edges': 60,
'fp_nodes': 0,
'ns_nodes': 0,
'ws_edges': 47},
'version': '0.3.1.dev47+g0c591a4'}]
Use an IOU matcher which supports a minimum threshold for overlap and run division metrics.
[7]:
iou_results, iou_matched = run_metrics(
gt_data=gt_data,
pred_data=pred_data,
matcher=IOUMatcher(iou_threshold=0.1, one_to_one=True),
metrics=[DivisionMetrics(max_frame_buffer=2)],
)
pp.pprint(iou_results)
Matching frames: 100%|██████████| 92/92 [00:00<00:00, 224.73it/s]
Node errors already calculated. Skipping graph annotation
Edge errors already calculated. Skipping graph annotation
[ { 'gt': 'Hela-01_GT',
'matcher': { '_matching_type': 'one-to-one',
'iou_threshold': 0.1,
'matching type': 'one-to-one',
'name': 'IOUMatcher',
'one_to_one': True},
'metric': { 'frame_buffer': 2,
'name': 'DivisionMetrics',
'relax_skips_gt': False,
'relax_skips_pred': False,
'valid_match_types': ['one-to-one']},
'pred': 'Hela-01_RES',
'results': { 'Frame Buffer 0': { 'Division F1': nan,
'Division Precision': 0.0,
'Division Recall': 0.0,
'False Negative Divisions': 19,
'False Positive Divisions': 31,
'Mitotic Branching Correctness': 0.0,
'Total GT Divisions': 94,
'Total Predicted Divisions': 106,
'True Positive Divisions': 0,
'Wrong Children Divisions': 75},
'Frame Buffer 1': { 'Division F1': nan,
'Division Precision': 0.0,
'Division Recall': 0.0,
'False Negative Divisions': 19,
'False Positive Divisions': 31,
'Mitotic Branching Correctness': 0.0,
'Total GT Divisions': 94,
'Total Predicted Divisions': 106,
'True Positive Divisions': 0,
'Wrong Children Divisions': 75},
'Frame Buffer 2': { 'Division F1': nan,
'Division Precision': 0.0,
'Division Recall': 0.0,
'False Negative Divisions': 19,
'False Positive Divisions': 31,
'Mitotic Branching Correctness': 0.0,
'Total GT Divisions': 94,
'Total Predicted Divisions': 106,
'True Positive Divisions': 0,
'Wrong Children Divisions': 75}},
'version': '0.3.1.dev47+g0c591a4'}]
Once errors are annotated and metrics are computed, you can save the annotated graphs and results to GEFF format using the export_graphs_to_geff utility.
[ ]:
export_graphs_to_geff(
"./traccuracy_results.zarr",
matched=iou_matched,
results=iou_results,
# if division metrics have been computed,
# you can optionally specify which frame buffer to use
# to re-annotate the division events
target_frame_buffer=1,
)