Coverage for collectra/evaluator/visualise.py: 28.43%
102 statements
« prev ^ index » next coverage.py v7.16.2, created at 2026-10-16 09:56 +0000
1__all__ = ["generate_evaluation_html"]
3from collections import defaultdict
4from pathlib import Path
5from typing import Any
7import plotly.colors
9from ..logger import get_logger
10from .base import EvaluationReport
12logger = get_logger(__name__)
14# ── Colour palette ────────────────────────────────────────────────────────────
15_PALETTE = plotly.colors.qualitative.Pastel
18def _color(i: int) -> str:
19 return _PALETTE[i % len(_PALETTE)]
22# ── Layout config ─────────────────────────────────────────────────────────────
23_EXCLUDED_TYPES = {"Image"} # data types to omit from the report
24_ROW_HEIGHT_PX = 600 # pixels per subplot row
25_VERTICAL_SPACING = 0.06 # fraction of figure height between rows
27# ── Scatter (Precision vs Recall) config ──────────────────────────────────────
28_BUBBLE_MIN = 6 # minimum bubble diameter (px)
29_BUBBLE_MAX = 24 # maximum bubble diameter (px)
30_BUBBLE_OPACITY = 0.85
31_SCATTER_RANGE = (0, 1.05) # axis range for both P and R
33# ── Box plot config ───────────────────────────────────────────────────────────
34_SCORE_RANGE = (0.0, 1.05) # y-axis range for match score
36# ── Legend config ─────────────────────────────────────────────────────────────
37_LEGEND_X = 1.02 # x anchor (fraction of figure width)
38_LEGEND_BGCOLOR = "rgba(30,30,30,0.8)"
39_LEGEND_BORDER = "rgba(255,255,255,0.15)"
41# ── Page chrome ───────────────────────────────────────────────────────────────
42_PAPER_BG = "#121212"
43_PLOT_BG = "#121212"
44_PAGE_BG = "#000000"
47# ── Artefact helpers ──────────────────────────────────────────────────────────────
50def _label_colors(report: EvaluationReport, data_types: list[str]) -> dict[str, str]:
51 """Assign a stable palette color to every label across all data types."""
52 ordered: list[str] = []
53 for dt in data_types:
54 for lbl in report.aggregate[dt].get("per_label", {}):
55 if lbl not in ordered:
56 ordered.append(lbl)
57 return {lbl: _color(i) for i, lbl in enumerate(ordered)}
60def _collect_scores(
61 report: EvaluationReport,
62 data_types: list[str],
63) -> dict[tuple[str, str], list[float]]:
64 """Collect per-match scores keyed by (data_type, label)."""
65 scores: dict[tuple[str, str], list[float]] = defaultdict(list)
66 allowed = set(data_types)
67 for fr in report.file_results:
68 for lbl, lm in fr.label_metrics.items():
69 for dt in lm.get("types", []):
70 if dt in allowed:
71 for m in lm.get("matches", []):
72 scores[(dt, lbl)].append(m["score"])
73 return scores
76# ── Trace builders ────────────────────────────────────────────────────────────
79def _scatter_traces(
80 per_label: dict,
81 legend_ref: str,
82 dt: str,
83 colors: dict[str, str],
84) -> list:
85 """One bubble trace per label for the Precision vs Recall plot."""
86 import plotly.graph_objects as go
88 supports = [v.get("support", 1) for v in per_label.values()]
89 max_support = max(supports, default=1)
91 traces = []
92 for lbl, metrics in per_label.items():
93 support = metrics.get("support", 1)
94 size = _BUBBLE_MIN + (_BUBBLE_MAX - _BUBBLE_MIN) * support / max_support
95 traces.append(
96 go.Scatter(
97 x=[metrics.get("recall", 0.0)],
98 y=[metrics.get("precision", 0.0)],
99 mode="markers",
100 name=lbl,
101 legendgroup=f"{dt}:{lbl}",
102 legend=legend_ref,
103 text=[f"{lbl}<br>support={support}"],
104 hovertemplate="%{text}<br>P=%{y:.3f} R=%{x:.3f}<extra></extra>",
105 marker=dict(size=size, opacity=_BUBBLE_OPACITY, color=colors[lbl]),
106 )
107 )
108 return traces
111def _box_traces(
112 labels: list[str],
113 scores: dict[tuple[str, str], list[float]],
114 legend_ref: str,
115 dt: str,
116 colors: dict[str, str],
117) -> list:
118 """One box trace per label for the Match Score distribution plot."""
119 import plotly.graph_objects as go
121 traces = []
122 for lbl in labels:
123 lbl_scores = scores.get((dt, lbl), [])
124 if lbl_scores:
125 traces.append(
126 go.Box(
127 y=lbl_scores,
128 name=lbl,
129 legendgroup=f"{dt}:{lbl}",
130 legend=legend_ref,
131 showlegend=False,
132 marker_color=colors[lbl],
133 boxmean=True,
134 )
135 )
136 return traces
139# ── Main entry point ──────────────────────────────────────────────────────────
142def generate_evaluation_html(report: EvaluationReport, output_path: Path) -> None:
143 """Build a dark-themed Plotly HTML report.
145 For each included data type, two subplot rows are produced:
146 - Precision vs Recall scatter bubble (per label)
147 - Match score box plot (per label)
148 """
149 try:
150 import plotly.graph_objects as go
151 import plotly.io as pio
152 from plotly.subplots import make_subplots
153 except ImportError:
154 logger.error(
155 "plotly is required for HTML reports. Install with: pip install plotly"
156 )
157 return
159 data_types = [dt for dt in report.aggregate if dt not in _EXCLUDED_TYPES]
161 if not data_types:
162 Path(output_path).write_text(
163 "<html><body><p>No evaluation data available.</p></body></html>",
164 encoding="utf-8",
165 )
166 return
168 colors = _label_colors(report, data_types)
169 scores = _collect_scores(report, data_types)
171 n_rows = 2 * len(data_types)
172 row_height = (1.0 - (n_rows - 1) * _VERTICAL_SPACING) / n_rows
174 subplot_titles = [
175 title
176 for dt in data_types
177 for title in (f"Precision vs Recall — {dt}", f"Match Score by Label — {dt}")
178 ]
180 fig = make_subplots(
181 rows=n_rows,
182 cols=1,
183 subplot_titles=subplot_titles,
184 vertical_spacing=_VERTICAL_SPACING,
185 )
186 fig.update_layout(
187 template="plotly_dark",
188 title_text="Collectra Evaluation Report",
189 height=_ROW_HEIGHT_PX * n_rows,
190 colorway=_PALETTE,
191 paper_bgcolor=_PAPER_BG,
192 plot_bgcolor=_PLOT_BG,
193 )
195 for dt_idx, dt in enumerate(data_types):
196 scatter_row = dt_idx * 2 + 1
197 box_row = dt_idx * 2 + 2
198 legend_ref = "legend" if dt_idx == 0 else f"legend{dt_idx + 1}"
199 per_label = report.aggregate[dt].get("per_label", {})
200 labels = list(per_label.keys())
202 # Legend — anchored to the top of this data type's section
203 legend_y = 1.0 - dt_idx * (2 * row_height + 2 * _VERTICAL_SPACING)
204 _legend_update: dict[str, Any] = {
205 legend_ref: dict(
206 title=dict(text=f"<b>{dt}</b>", side="top center"),
207 x=_LEGEND_X,
208 y=legend_y,
209 yanchor="top",
210 bgcolor=_LEGEND_BGCOLOR,
211 bordercolor=_LEGEND_BORDER,
212 borderwidth=1,
213 itemsizing="constant",
214 )
215 }
216 fig.update_layout(**_legend_update)
218 for trace in _scatter_traces(per_label, legend_ref, dt, colors):
219 fig.add_trace(trace, row=scatter_row, col=1)
220 fig.update_xaxes(
221 title_text="Recall", range=list(_SCATTER_RANGE), row=scatter_row, col=1
222 )
223 fig.update_yaxes(
224 title_text="Precision", range=list(_SCATTER_RANGE), row=scatter_row, col=1
225 )
227 for trace in _box_traces(labels, scores, legend_ref, dt, colors):
228 fig.add_trace(trace, row=box_row, col=1)
229 fig.update_yaxes(
230 title_text="Match Score", range=list(_SCORE_RANGE), row=box_row, col=1
231 )
233 _include_plotlyjs: Any = "cdn"
234 html_fig = pio.to_html(fig, include_plotlyjs=_include_plotlyjs, full_html=False)
236 full_html = f"""<!DOCTYPE html>
237<html>
238<head>
239 <meta charset="utf-8">
240 <title>Collectra Evaluation Report</title>
241 <style>
242 * {{ box-sizing: border-box; }}
243 body {{
244 font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
245 margin: 0;
246 padding: 20px;
247 background: {_PAGE_BG};
248 color: #e6e6e6;
249 }}
250 h1 {{
251 margin: 0 0 20px 0;
252 font-size: 20px;
253 color: #e6e6e6;
254 border-bottom: 1px solid #2a2a2a;
255 padding-bottom: 12px;
256 }}
257 .report-container {{
258 background: {_PAPER_BG};
259 padding: 20px;
260 border-radius: 8px;
261 box-shadow: 0 2px 10px rgba(0,0,0,0.5);
262 }}
263 </style>
264</head>
265<body>
266 <h1>Collectra Evaluation Report</h1>
267 <div class="report-container">
268 {html_fig}
269 </div>
270</body>
271</html>"""
273 output_path = Path(output_path)
274 output_path.parent.mkdir(parents=True, exist_ok=True)
275 output_path.write_text(full_html, encoding="utf-8")
276 logger.info("HTML evaluation report written to %s", output_path)