diff --git a/README.md b/README.md index 6b9f3a9..867c239 100644 --- a/README.md +++ b/README.md @@ -8,18 +8,38 @@ MVTec-style folders and the original VisA layout. | Path | Purpose | | --- | --- | -| `train.py` | Shared training and final-evaluation command-line entry point | +| `train.py`, `anomaly_detection/__main__.py` | Script and `python -m` entry points for the same training pipeline | +| `anomaly_detection/cli.py` | Shared argument parsing for YAML and the original training flags | | `inference.py` | FastFlow inference, metrics and visualizations | | `ssn_inference.py` | SuperSimpleNet inference, metrics and visualizations | +| `anomaly_detection/inference_utils.py` | Shared AUROC/AUPRO metrics, calibration loading and MVTec file helpers | | `anomaly_detection/data.py` | MVTec, MVTec-style and VisA dataset loading | -| `anomaly_detection/modeling/` | FastFlow and SuperSimpleNet implementations | -| `anomaly_detection/training/` | Model-specific training and evaluation utilities | +| `anomaly_detection/modeling/` | Detector implementations in `fastflow.py` and `supersimplenet.py` | +| `anomaly_detection/training/fastflow.py`, `anomaly_detection/training/supersimplenet.py` | Model-specific optimizers, training, calibration and final evaluation | +| `anomaly_detection/training/losses.py` | FastFlow NLL, SSN focal and SSN BCE objectives | | `anomaly_detection/builder.py` | Configuration-driven model, loss and trainer factories | | `anomaly_detection/config.py` | YAML schema, defaults and validation | | `anomaly_detection/pipeline.py` | Shared experiment lifecycle | -| `configs/` | Ready-to-edit FastFlow, SSN focal and SSN BCE YAML experiments | -| `docs/yaml_training.md` | YAML options, supported combinations and extension guide | -| `tests/` | Regression tests for evaluation and model behavior | +| `configs/fastflow.yaml` | FastFlow experiment with its NLL loss | +| `configs/ssn.yaml` | SSN experiment with its focal loss | +| `configs/ssn_bce.yaml` | SSN experiment with the alternative BCE loss | +| `README.md`, `docs/yaml_training.md` | Quick start and commands; detailed YAML reference and extension guide | +| `tests/test_config.py` | Configuration validation and command-line error handling | +| `tests/test_regressions.py` | Model contracts, calibration, inference metrics and file handling | +| `tests/test_yaml_training.py` | Component selection and offline training/checkpoint/inference integration | +| `requirements.txt`, `requirements-dev.txt` | Runtime dependencies; runtime dependencies plus test and static-check tools | +| `pyproject.toml` | Pytest discovery and reporting settings | +| `.github/workflows/tests.yml` | Automated CPU tests and static checks on pushes and pull requests | +| `.editorconfig` | Consistent indentation, encoding and line endings | +| `.gitignore` | Excludes local environments, datasets, caches and generated outputs | + +The three `__init__.py` files define the package boundaries and public model and +trainer imports. The training entry points share one implementation; the two +inference scripts retain their model-specific checkpoint handling and overlays. + +For a new experiment, copy a file in `configs/` and edit the YAML. Reusable Python +code belongs in `anomaly_detection/`, with checks in `tests/` and usage details in +`docs/`. Keep downloaded datasets, checkpoints and generated images out of Git. ## Installation diff --git a/anomaly_detection/inference_utils.py b/anomaly_detection/inference_utils.py new file mode 100644 index 0000000..34a8baf --- /dev/null +++ b/anomaly_detection/inference_utils.py @@ -0,0 +1,351 @@ +"""Shared metrics, calibration and MVTec file helpers for inference.""" + +import json +from dataclasses import dataclass +from pathlib import Path +from typing import List, Optional, Tuple + +import cv2 +import numpy as np +import torch + + +# ============================================================ +# Self-implemented AUPRO (minimal, compute-only, no torchmetrics) +# ============================================================ +class SelfAUPRO: + """ + Minimal AUPRO implementation inspired by anomalib's AUPRO: + - connected components on GT mask + - compute PRO curve (FPR vs averaged per-region overlap) + - integrate area up to fpr_limit and normalize by fpr_limit + + Expected: + preds: (B,H,W) or (B,1,H,W) float in [0,1] (higher => more anomalous) + target: (B,H,W) or (B,1,H,W) binary {0,1} or {0,255} + """ + + def __init__(self, fpr_limit: float = 0.3): + self.fpr_limit = float(fpr_limit) + self._preds: List[torch.Tensor] = [] + self._target: List[torch.Tensor] = [] + + def update(self, preds: torch.Tensor, target: torch.Tensor) -> None: + if preds.dim() == 4 and preds.size(1) == 1: + preds = preds.squeeze(1) + if target.dim() == 4 and target.size(1) == 1: + target = target.squeeze(1) + + # store on CPU to reduce GPU memory + preds = preds.detach().float().cpu() + target = target.detach().float().cpu() + + # normalize target into {0,1} + if target.max() > 1.0: + target = (target > 0).float() + else: + target = (target > 0.5).float() + + # clamp preds + preds = preds.clamp(0.0, 1.0) + + self._preds.append(preds) + self._target.append(target) + + @staticmethod + def _cca_cv2(mask01: np.ndarray) -> np.ndarray: + """Connected components for one image. mask01 is {0,1} uint8.""" + mask01 = (mask01 > 0).astype(np.uint8) + # returns labels in [0..N] + _, labels = cv2.connectedComponents(mask01, connectivity=8) + return labels.astype(np.int32) + + @staticmethod + def _make_global_region_labels(cca_bhw: torch.Tensor) -> torch.Tensor: + """Offset connected component labels across batch to make them unique (except 0 background).""" + cca_off = cca_bhw.clone() + current_offset = 0 + B = int(cca_off.size(0)) + for b in range(B): + img = cca_off[b] + uniq = torch.unique(img) + uniq_fg = uniq[uniq != 0] + num_regions = int(uniq_fg.numel()) + if num_regions == 0: + continue + fg = img > 0 + img[fg] = img[fg] + current_offset + cca_off[b] = img + current_offset += num_regions + return cca_off + + def perform_cca(self) -> torch.Tensor: + """Return (B,H,W) integer labels; 0 is background; >0 are region IDs unique across batch.""" + target = torch.cat(self._target, dim=0) # (B,H,W) + if target.min() < 0 or target.max() > 1: + raise ValueError(f"AUPRO expects target in [0,1], got [{float(target.min())},{float(target.max())}]") + + # CPU CCA via OpenCV per image + target_np = target.numpy() + ccas = [] + for b in range(target_np.shape[0]): + labels = self._cca_cv2(target_np[b]) + ccas.append(labels) + cca = torch.from_numpy(np.stack(ccas, axis=0)) # (B,H,W) int32 + cca = self._make_global_region_labels(cca) + return cca.long() + + @staticmethod + def _auc_trapz(x: torch.Tensor, y: torch.Tensor) -> torch.Tensor: + """Trapezoidal integration (assumes x sorted ascending).""" + if x.numel() < 2: + return torch.tensor(0.0, device=x.device, dtype=torch.float32) + dx = x[1:] - x[:-1] + avg = 0.5 * (y[1:] + y[:-1]) + return torch.sum(dx * avg) + + def compute_pro(self, cca: torch.Tensor, preds: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Compute PRO curve (FPR vs averaged per-region overlap). + cca: (B,H,W) int labels, 0 background + preds: (B,H,W) float, higher => more anomalous + """ + device = preds.device + + labels = cca.reshape(-1).long() + preds_flat = preds.reshape(-1).float() + + background = labels == 0 + fp_change = background.float() + num_bg = fp_change.sum() + + f_lim = float(self.fpr_limit) + + if num_bg <= 0: + return ( + torch.tensor([0.0, f_lim], device=device), + torch.tensor([0.0, 0.0], device=device), + ) + + max_label = int(labels.max().item()) + if max_label == 0: + return ( + torch.tensor([0.0, f_lim], device=device), + torch.tensor([0.0, 0.0], device=device), + ) + + region_sizes = torch.bincount(labels, minlength=max_label + 1).float() + num_regions = (region_sizes[1:] > 0).sum() + + if num_regions <= 0: + return ( + torch.tensor([0.0, f_lim], device=device), + torch.tensor([0.0, 0.0], device=device), + ) + + fg_mask = labels > 0 + pro_change = torch.zeros_like(preds_flat) + pro_change[fg_mask] = 1.0 / region_sizes[labels[fg_mask]] + + idx = torch.argsort(preds_flat, descending=True) + fp_sorted = fp_change[idx] + pro_sorted = pro_change[idx] + preds_sorted = preds_flat[idx] + + fpr = torch.cumsum(fp_sorted, 0) / num_bg + pro = torch.cumsum(pro_sorted, 0) / num_regions + fpr = torch.clamp(fpr, max=1.0) + pro = torch.clamp(pro, max=1.0) + + # remove duplicate thresholds + keep = torch.ones_like(preds_sorted, dtype=torch.bool) + keep[:-1] = preds_sorted[:-1] != preds_sorted[1:] + fpr = fpr[keep] + pro = pro[keep] + + # prepend zero + fpr = torch.cat([torch.tensor([0.0], device=device), fpr]) + pro = torch.cat([torch.tensor([0.0], device=device), pro]) + + # clip at fpr_limit with linear interpolation + mask = fpr <= f_lim + if mask.any(): + i = int(mask.nonzero(as_tuple=True)[0][-1].item()) + if fpr[i] < f_lim and i + 1 < fpr.numel(): + f1, f2 = fpr[i], fpr[i + 1] + p1, p2 = pro[i], pro[i + 1] + p_lim = p1 + (p2 - p1) * (f_lim - f1) / (f2 - f1 + 1e-12) + fpr = torch.cat([fpr[: i + 1], torch.tensor([f_lim], device=device)]) + pro = torch.cat([pro[: i + 1], torch.tensor([p_lim], device=device)]) + else: + fpr = fpr[: i + 1] + pro = pro[: i + 1] + else: + fpr = torch.tensor([0.0, f_lim], device=device) + pro = torch.tensor([0.0, 0.0], device=device) + + return fpr, pro + + def compute(self) -> torch.Tensor: + if not self._preds: + return torch.tensor(0.0) + + cca = self.perform_cca() # (B,H,W) on CPU + preds = torch.cat(self._preds, dim=0) # (B,H,W) on CPU + + # compute PRO curve on CPU tensors + fpr, pro = self.compute_pro(cca=cca, preds=preds) + + area = self._auc_trapz(fpr, pro) + # normalize by fpr_limit (or by last x) + denom = fpr[-1].clamp_min(1e-12) + return area / denom + + +# ----------------------------- +# Utilities: metrics (no sklearn) +# ----------------------------- +def confusion_counts(y_true: np.ndarray, y_pred: np.ndarray): + y_true = y_true.astype(np.int64) + y_pred = y_pred.astype(np.int64) + tp = int(((y_true == 1) & (y_pred == 1)).sum()) + tn = int(((y_true == 0) & (y_pred == 0)).sum()) + fp = int(((y_true == 0) & (y_pred == 1)).sum()) + fn = int(((y_true == 1) & (y_pred == 0)).sum()) + return tp, tn, fp, fn + + +def precision_from_counts(tp: int, fp: int) -> float: + return float(tp / (tp + fp + 1e-12)) + + +def accuracy_from_counts(tp: int, tn: int, fp: int, fn: int) -> float: + return float((tp + tn) / (tp + tn + fp + fn + 1e-12)) + + +def auroc_from_scores(y_true: np.ndarray, y_score: np.ndarray) -> float: + """AUROC via Mann–Whitney U (rank-based), tie-safe.""" + y_true = y_true.astype(np.int64) + y_score = y_score.astype(np.float64) + + pos = (y_true == 1) + neg = (y_true == 0) + n_pos = int(pos.sum()) + n_neg = int(neg.sum()) + if n_pos == 0 or n_neg == 0: + return float("nan") + + order = np.argsort(y_score, kind="mergesort") + ranks = np.empty_like(order, dtype=np.float64) + ranks[order] = np.arange(1, len(y_score) + 1, dtype=np.float64) + + sorted_scores = y_score[order] + i = 0 + while i < len(sorted_scores): + j = i + while j + 1 < len(sorted_scores) and sorted_scores[j + 1] == sorted_scores[i]: + j += 1 + if j > i: + avg_rank = (i + 1 + j + 1) / 2.0 + ranks[order[i : j + 1]] = avg_rank + i = j + 1 + + sum_ranks_pos = float(ranks[pos].sum()) + u = sum_ranks_pos - n_pos * (n_pos + 1) / 2.0 + auc = u / (n_pos * n_neg) + return float(auc) + + +def load_calibration(checkpoint_path: str, calibration_path: Optional[str]) -> dict: + candidates = [] + if calibration_path: + candidates.append(Path(calibration_path)) + checkpoint = Path(checkpoint_path) + candidates.extend( + [checkpoint.parent / "calibration.json", checkpoint.parent.parent / "calibration.json"] + ) + for candidate in candidates: + if candidate.is_file(): + with candidate.open("r", encoding="utf-8") as f: + calibration = json.load(f) + if "image_threshold" not in calibration or "pixel_threshold" not in calibration: + raise ValueError(f"Invalid calibration file: {candidate}") + print(f"Loaded calibration: {candidate}") + return calibration + raise FileNotFoundError( + "No calibration.json found. Run train.py with the updated pipeline or pass --calibration_path." + ) + + +# ----------------------------- +# Data helpers +# ----------------------------- +def iter_images_recursive(root: Path): + exts = {".png", ".jpg", ".jpeg", ".bmp", ".tif", ".tiff"} + for p in root.rglob("*"): + if not p.is_file(): + continue + if p.suffix.lower() not in exts: + continue + if "ground_truth" in {x.lower() for x in p.parts}: + continue + yield p + + +def infer_gt_label_from_path(p: Path) -> Optional[int]: + parts = [x.lower() for x in p.parts] + if "test" not in parts: + return None + i = parts.index("test") + if i + 1 >= len(parts): + return None + return 0 if parts[i + 1] == "good" else 1 + + +def defect_name_from_path(p: Path) -> str: + parts = list(p.parts) + lower = [x.lower() for x in parts] + if "test" in lower: + i = lower.index("test") + if i + 1 < len(parts): + return parts[i + 1] + return p.parent.name + + +def load_gt_mask_mvtec(img_path: Path) -> Optional[np.ndarray]: + parts = list(img_path.parts) + lower = [p.lower() for p in parts] + if "test" not in lower: + return None + ti = lower.index("test") + if ti + 1 >= len(parts): + return None + + defect = parts[ti + 1] + if defect.lower() == "good": + return None + + cat_dir = img_path.parents[2] # .../ + gt_path = cat_dir / "ground_truth" / defect / f"{img_path.stem}_mask.png" + if not gt_path.exists(): + gt_path = cat_dir / "ground_truth" / defect / f"{img_path.stem}.png" + if not gt_path.exists(): + return None + + m = cv2.imread(str(gt_path), cv2.IMREAD_GRAYSCALE) + if m is None: + return None + return ((m > 0).astype(np.uint8) * 255) + + +# ----------------------------- +# Collected prediction +# ----------------------------- +@dataclass +class Sample: + path: Path + gt_label: Optional[int] + score_raw: float + amap_hw: np.ndarray + orig_rgb: np.ndarray + gt_mask_orig: Optional[np.ndarray] diff --git a/anomaly_detection/training/fastflow.py b/anomaly_detection/training/fastflow.py index e35d386..4bcdd84 100644 --- a/anomaly_detection/training/fastflow.py +++ b/anomaly_detection/training/fastflow.py @@ -1,4 +1,5 @@ -# trainer.py +"""FastFlow training, validation calibration and final evaluation.""" + from pathlib import Path from typing import Any, Dict, Optional diff --git a/anomaly_detection/training/supersimplenet.py b/anomaly_detection/training/supersimplenet.py index 4267c90..de51945 100644 --- a/anomaly_detection/training/supersimplenet.py +++ b/anomaly_detection/training/supersimplenet.py @@ -1,10 +1,4 @@ -# ssntrainer.py -# Updated anomalib-style SSN trainer/evaluator with: -# - Stable truncation loss (applies on sigmoid(map_logits)) -# - Validation-loss checkpoint selection with an untouched test set -# - Fixed anomaly-score direction (higher means more anomalous) -# - Mask alignment safety (resizes masks to pred_map size with NEAREST) -# - Accuracy and precision from validation-calibrated fixed thresholds +"""SuperSimpleNet training, validation calibration and final evaluation.""" from __future__ import annotations diff --git a/inference.py b/inference.py index 3d95c7f..2d36586 100644 --- a/inference.py +++ b/inference.py @@ -6,9 +6,7 @@ # - contour overlay drawn on ORIGINAL image, but thresholding done in crop-space then "uncropped" back import argparse -import json import math -from dataclasses import dataclass from pathlib import Path from typing import List, Optional, Tuple @@ -17,350 +15,22 @@ import torch from PIL import Image +from anomaly_detection.inference_utils import ( + Sample, + SelfAUPRO, + accuracy_from_counts, + auroc_from_scores, + confusion_counts, + defect_name_from_path, + infer_gt_label_from_path, + iter_images_recursive, + load_calibration, + load_gt_mask_mvtec, + precision_from_counts, +) from anomaly_detection.modeling import FastFlowModel -# ============================================================ -# Self-implemented AUPRO (minimal, compute-only, no torchmetrics) -# ============================================================ -class SelfAUPRO: - """ - Minimal AUPRO implementation inspired by anomalib's AUPRO: - - connected components on GT mask - - compute PRO curve (FPR vs averaged per-region overlap) - - integrate area up to fpr_limit and normalize by fpr_limit - - Expected: - preds: (B,H,W) or (B,1,H,W) float in [0,1] (higher => more anomalous) - target: (B,H,W) or (B,1,H,W) binary {0,1} or {0,255} - """ - - def __init__(self, fpr_limit: float = 0.3): - self.fpr_limit = float(fpr_limit) - self._preds: List[torch.Tensor] = [] - self._target: List[torch.Tensor] = [] - - def update(self, preds: torch.Tensor, target: torch.Tensor) -> None: - if preds.dim() == 4 and preds.size(1) == 1: - preds = preds.squeeze(1) - if target.dim() == 4 and target.size(1) == 1: - target = target.squeeze(1) - - # store on CPU to reduce GPU memory - preds = preds.detach().float().cpu() - target = target.detach().float().cpu() - - # normalize target into {0,1} - if target.max() > 1.0: - target = (target > 0).float() - else: - target = (target > 0.5).float() - - # clamp preds - preds = preds.clamp(0.0, 1.0) - - self._preds.append(preds) - self._target.append(target) - - @staticmethod - def _cca_cv2(mask01: np.ndarray) -> np.ndarray: - """Connected components for one image. mask01 is {0,1} uint8.""" - mask01 = (mask01 > 0).astype(np.uint8) - # returns labels in [0..N] - _, labels = cv2.connectedComponents(mask01, connectivity=8) - return labels.astype(np.int32) - - @staticmethod - def _make_global_region_labels(cca_bhw: torch.Tensor) -> torch.Tensor: - """Offset connected component labels across batch to make them unique (except 0 background).""" - cca_off = cca_bhw.clone() - current_offset = 0 - B = int(cca_off.size(0)) - for b in range(B): - img = cca_off[b] - uniq = torch.unique(img) - uniq_fg = uniq[uniq != 0] - num_regions = int(uniq_fg.numel()) - if num_regions == 0: - continue - fg = img > 0 - img[fg] = img[fg] + current_offset - cca_off[b] = img - current_offset += num_regions - return cca_off - - def perform_cca(self) -> torch.Tensor: - """Return (B,H,W) integer labels; 0 is background; >0 are region IDs unique across batch.""" - target = torch.cat(self._target, dim=0) # (B,H,W) - if target.min() < 0 or target.max() > 1: - raise ValueError(f"AUPRO expects target in [0,1], got [{float(target.min())},{float(target.max())}]") - - # CPU CCA via OpenCV per image - target_np = target.numpy() - ccas = [] - for b in range(target_np.shape[0]): - labels = self._cca_cv2(target_np[b]) - ccas.append(labels) - cca = torch.from_numpy(np.stack(ccas, axis=0)) # (B,H,W) int32 - cca = self._make_global_region_labels(cca) - return cca.long() - - @staticmethod - def _auc_trapz(x: torch.Tensor, y: torch.Tensor) -> torch.Tensor: - """Trapezoidal integration (assumes x sorted ascending).""" - if x.numel() < 2: - return torch.tensor(0.0, device=x.device, dtype=torch.float32) - dx = x[1:] - x[:-1] - avg = 0.5 * (y[1:] + y[:-1]) - return torch.sum(dx * avg) - - def compute_pro(self, cca: torch.Tensor, preds: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: - """ - Compute PRO curve (FPR vs averaged per-region overlap). - cca: (B,H,W) int labels, 0 background - preds: (B,H,W) float, higher => more anomalous - """ - device = preds.device - - labels = cca.reshape(-1).long() - preds_flat = preds.reshape(-1).float() - - background = labels == 0 - fp_change = background.float() - num_bg = fp_change.sum() - - f_lim = float(self.fpr_limit) - - if num_bg <= 0: - return ( - torch.tensor([0.0, f_lim], device=device), - torch.tensor([0.0, 0.0], device=device), - ) - - max_label = int(labels.max().item()) - if max_label == 0: - return ( - torch.tensor([0.0, f_lim], device=device), - torch.tensor([0.0, 0.0], device=device), - ) - - region_sizes = torch.bincount(labels, minlength=max_label + 1).float() - num_regions = (region_sizes[1:] > 0).sum() - - if num_regions <= 0: - return ( - torch.tensor([0.0, f_lim], device=device), - torch.tensor([0.0, 0.0], device=device), - ) - - fg_mask = labels > 0 - pro_change = torch.zeros_like(preds_flat) - pro_change[fg_mask] = 1.0 / region_sizes[labels[fg_mask]] - - idx = torch.argsort(preds_flat, descending=True) - fp_sorted = fp_change[idx] - pro_sorted = pro_change[idx] - preds_sorted = preds_flat[idx] - - fpr = torch.cumsum(fp_sorted, 0) / num_bg - pro = torch.cumsum(pro_sorted, 0) / num_regions - fpr = torch.clamp(fpr, max=1.0) - pro = torch.clamp(pro, max=1.0) - - # remove duplicate thresholds - keep = torch.ones_like(preds_sorted, dtype=torch.bool) - keep[:-1] = preds_sorted[:-1] != preds_sorted[1:] - fpr = fpr[keep] - pro = pro[keep] - - # prepend zero - fpr = torch.cat([torch.tensor([0.0], device=device), fpr]) - pro = torch.cat([torch.tensor([0.0], device=device), pro]) - - # clip at fpr_limit with linear interpolation - mask = fpr <= f_lim - if mask.any(): - i = int(mask.nonzero(as_tuple=True)[0][-1].item()) - if fpr[i] < f_lim and i + 1 < fpr.numel(): - f1, f2 = fpr[i], fpr[i + 1] - p1, p2 = pro[i], pro[i + 1] - p_lim = p1 + (p2 - p1) * (f_lim - f1) / (f2 - f1 + 1e-12) - fpr = torch.cat([fpr[: i + 1], torch.tensor([f_lim], device=device)]) - pro = torch.cat([pro[: i + 1], torch.tensor([p_lim], device=device)]) - else: - fpr = fpr[: i + 1] - pro = pro[: i + 1] - else: - fpr = torch.tensor([0.0, f_lim], device=device) - pro = torch.tensor([0.0, 0.0], device=device) - - return fpr, pro - - def compute(self) -> torch.Tensor: - if not self._preds: - return torch.tensor(0.0) - - cca = self.perform_cca() # (B,H,W) on CPU - preds = torch.cat(self._preds, dim=0) # (B,H,W) on CPU - - # compute PRO curve on CPU tensors - fpr, pro = self.compute_pro(cca=cca, preds=preds) - - area = self._auc_trapz(fpr, pro) - # normalize by fpr_limit (or by last x) - denom = fpr[-1].clamp_min(1e-12) - return area / denom - - -# ----------------------------- -# Utilities: metrics (no sklearn) -# ----------------------------- -def confusion_counts(y_true: np.ndarray, y_pred: np.ndarray): - y_true = y_true.astype(np.int64) - y_pred = y_pred.astype(np.int64) - tp = int(((y_true == 1) & (y_pred == 1)).sum()) - tn = int(((y_true == 0) & (y_pred == 0)).sum()) - fp = int(((y_true == 0) & (y_pred == 1)).sum()) - fn = int(((y_true == 1) & (y_pred == 0)).sum()) - return tp, tn, fp, fn - - -def precision_from_counts(tp: int, fp: int) -> float: - return float(tp / (tp + fp + 1e-12)) - - -def accuracy_from_counts(tp: int, tn: int, fp: int, fn: int) -> float: - return float((tp + tn) / (tp + tn + fp + fn + 1e-12)) - - -def auroc_from_scores(y_true: np.ndarray, y_score: np.ndarray) -> float: - """AUROC via Mann–Whitney U (rank-based), tie-safe.""" - y_true = y_true.astype(np.int64) - y_score = y_score.astype(np.float64) - - pos = (y_true == 1) - neg = (y_true == 0) - n_pos = int(pos.sum()) - n_neg = int(neg.sum()) - if n_pos == 0 or n_neg == 0: - return float("nan") - - order = np.argsort(y_score, kind="mergesort") - ranks = np.empty_like(order, dtype=np.float64) - ranks[order] = np.arange(1, len(y_score) + 1, dtype=np.float64) - - sorted_scores = y_score[order] - i = 0 - while i < len(sorted_scores): - j = i - while j + 1 < len(sorted_scores) and sorted_scores[j + 1] == sorted_scores[i]: - j += 1 - if j > i: - avg_rank = (i + 1 + j + 1) / 2.0 - ranks[order[i : j + 1]] = avg_rank - i = j + 1 - - sum_ranks_pos = float(ranks[pos].sum()) - u = sum_ranks_pos - n_pos * (n_pos + 1) / 2.0 - auc = u / (n_pos * n_neg) - return float(auc) - - -def load_calibration(checkpoint_path: str, calibration_path: Optional[str]) -> dict: - candidates = [] - if calibration_path: - candidates.append(Path(calibration_path)) - checkpoint = Path(checkpoint_path) - candidates.extend( - [checkpoint.parent / "calibration.json", checkpoint.parent.parent / "calibration.json"] - ) - for candidate in candidates: - if candidate.is_file(): - with candidate.open("r", encoding="utf-8") as f: - calibration = json.load(f) - if "image_threshold" not in calibration or "pixel_threshold" not in calibration: - raise ValueError(f"Invalid calibration file: {candidate}") - print(f"Loaded calibration: {candidate}") - return calibration - raise FileNotFoundError( - "No calibration.json found. Run train.py with the updated pipeline or pass --calibration_path." - ) - - -# ----------------------------- -# Data helpers -# ----------------------------- -def iter_images_recursive(root: Path): - exts = {".png", ".jpg", ".jpeg", ".bmp", ".tif", ".tiff"} - for p in root.rglob("*"): - if not p.is_file(): - continue - if p.suffix.lower() not in exts: - continue - if "ground_truth" in {x.lower() for x in p.parts}: - continue - yield p - - -def infer_gt_label_from_path(p: Path) -> Optional[int]: - parts = [x.lower() for x in p.parts] - if "test" not in parts: - return None - i = parts.index("test") - if i + 1 >= len(parts): - return None - return 0 if parts[i + 1] == "good" else 1 - - -def defect_name_from_path(p: Path) -> str: - parts = list(p.parts) - lower = [x.lower() for x in parts] - if "test" in lower: - i = lower.index("test") - if i + 1 < len(parts): - return parts[i + 1] - return p.parent.name - - -def load_gt_mask_mvtec(img_path: Path) -> Optional[np.ndarray]: - parts = list(img_path.parts) - lower = [p.lower() for p in parts] - if "test" not in lower: - return None - ti = lower.index("test") - if ti + 1 >= len(parts): - return None - - defect = parts[ti + 1] - if defect.lower() == "good": - return None - - cat_dir = img_path.parents[2] # .../ - gt_path = cat_dir / "ground_truth" / defect / f"{img_path.stem}_mask.png" - if not gt_path.exists(): - gt_path = cat_dir / "ground_truth" / defect / f"{img_path.stem}.png" - if not gt_path.exists(): - return None - - m = cv2.imread(str(gt_path), cv2.IMREAD_GRAYSCALE) - if m is None: - return None - return ((m > 0).astype(np.uint8) * 255) - - -# ----------------------------- -# Core OOP runner -# ----------------------------- -@dataclass -class Sample: - path: Path - gt_label: Optional[int] - score_raw: float - amap_hw: np.ndarray - orig_rgb: np.ndarray - gt_mask_orig: Optional[np.ndarray] - - class FastFlowInferenceEngine: def __init__( self, diff --git a/ssn_inference.py b/ssn_inference.py index 7e83495..9ee6fe9 100644 --- a/ssn_inference.py +++ b/ssn_inference.py @@ -6,9 +6,7 @@ # - contour overlay drawn on ORIGINAL image, but thresholding done in crop-space then "uncropped" back import argparse -import json import math -from dataclasses import dataclass from pathlib import Path from typing import List, Optional, Tuple @@ -17,320 +15,22 @@ import torch from PIL import Image +from anomaly_detection.inference_utils import ( + Sample, + SelfAUPRO, + accuracy_from_counts, + auroc_from_scores, + confusion_counts, + defect_name_from_path, + infer_gt_label_from_path, + iter_images_recursive, + load_calibration, + load_gt_mask_mvtec, + precision_from_counts, +) from anomaly_detection.modeling import SuperSimpleNetModel -# ============================================================ -# Self-implemented AUPRO (minimal, compute-only, no torchmetrics) -# ============================================================ -class SelfAUPRO: - """ - Minimal AUPRO implementation inspired by anomalib's AUPRO: - - connected components on GT mask - - compute PRO curve (FPR vs averaged per-region overlap) - - integrate area up to fpr_limit and normalize by fpr_limit - - Expected: - preds: (B,H,W) or (B,1,H,W) float in [0,1] (higher => more anomalous) - target: (B,H,W) or (B,1,H,W) binary {0,1} or {0,255} - """ - - def __init__(self, fpr_limit: float = 0.3): - self.fpr_limit = float(fpr_limit) - self._preds: List[torch.Tensor] = [] - self._target: List[torch.Tensor] = [] - - def update(self, preds: torch.Tensor, target: torch.Tensor) -> None: - if preds.dim() == 4 and preds.size(1) == 1: - preds = preds.squeeze(1) - if target.dim() == 4 and target.size(1) == 1: - target = target.squeeze(1) - - preds = preds.detach().float().cpu() - target = target.detach().float().cpu() - - # normalize target into {0,1} - if target.max() > 1.0: - target = (target > 0).float() - else: - target = (target > 0.5).float() - - preds = preds.clamp(0.0, 1.0) - - self._preds.append(preds) - self._target.append(target) - - @staticmethod - def _cca_cv2(mask01: np.ndarray) -> np.ndarray: - mask01 = (mask01 > 0).astype(np.uint8) - _, labels = cv2.connectedComponents(mask01, connectivity=8) - return labels.astype(np.int32) - - @staticmethod - def _make_global_region_labels(cca_bhw: torch.Tensor) -> torch.Tensor: - cca_off = cca_bhw.clone() - current_offset = 0 - B = int(cca_off.size(0)) - for b in range(B): - img = cca_off[b] - uniq = torch.unique(img) - uniq_fg = uniq[uniq != 0] - num_regions = int(uniq_fg.numel()) - if num_regions == 0: - continue - fg = img > 0 - img[fg] = img[fg] + current_offset - cca_off[b] = img - current_offset += num_regions - return cca_off - - def perform_cca(self) -> torch.Tensor: - target = torch.cat(self._target, dim=0) # (B,H,W) - if target.min() < 0 or target.max() > 1: - raise ValueError(f"AUPRO expects target in [0,1], got [{float(target.min())},{float(target.max())}]") - - target_np = target.numpy() - ccas = [] - for b in range(target_np.shape[0]): - labels = self._cca_cv2(target_np[b]) - ccas.append(labels) - cca = torch.from_numpy(np.stack(ccas, axis=0)) - cca = self._make_global_region_labels(cca) - return cca.long() - - @staticmethod - def _auc_trapz(x: torch.Tensor, y: torch.Tensor) -> torch.Tensor: - if x.numel() < 2: - return torch.tensor(0.0, device=x.device, dtype=torch.float32) - dx = x[1:] - x[:-1] - avg = 0.5 * (y[1:] + y[:-1]) - return torch.sum(dx * avg) - - def compute_pro(self, cca: torch.Tensor, preds: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: - device = preds.device - labels = cca.reshape(-1).long() - preds_flat = preds.reshape(-1).float() - - background = labels == 0 - fp_change = background.float() - num_bg = fp_change.sum() - - f_lim = float(self.fpr_limit) - if num_bg <= 0: - return (torch.tensor([0.0, f_lim], device=device), torch.tensor([0.0, 0.0], device=device)) - - max_label = int(labels.max().item()) - if max_label == 0: - return (torch.tensor([0.0, f_lim], device=device), torch.tensor([0.0, 0.0], device=device)) - - region_sizes = torch.bincount(labels, minlength=max_label + 1).float() - num_regions = (region_sizes[1:] > 0).sum() - if num_regions <= 0: - return (torch.tensor([0.0, f_lim], device=device), torch.tensor([0.0, 0.0], device=device)) - - fg_mask = labels > 0 - pro_change = torch.zeros_like(preds_flat) - pro_change[fg_mask] = 1.0 / region_sizes[labels[fg_mask]] - - idx = torch.argsort(preds_flat, descending=True) - fp_sorted = fp_change[idx] - pro_sorted = pro_change[idx] - preds_sorted = preds_flat[idx] - - fpr = torch.cumsum(fp_sorted, 0) / num_bg - pro = torch.cumsum(pro_sorted, 0) / num_regions - fpr = torch.clamp(fpr, max=1.0) - pro = torch.clamp(pro, max=1.0) - - keep = torch.ones_like(preds_sorted, dtype=torch.bool) - keep[:-1] = preds_sorted[:-1] != preds_sorted[1:] - fpr = fpr[keep] - pro = pro[keep] - - fpr = torch.cat([torch.tensor([0.0], device=device), fpr]) - pro = torch.cat([torch.tensor([0.0], device=device), pro]) - - mask = fpr <= f_lim - if mask.any(): - i = int(mask.nonzero(as_tuple=True)[0][-1].item()) - if fpr[i] < f_lim and i + 1 < fpr.numel(): - f1, f2 = fpr[i], fpr[i + 1] - p1, p2 = pro[i], pro[i + 1] - p_lim = p1 + (p2 - p1) * (f_lim - f1) / (f2 - f1 + 1e-12) - fpr = torch.cat([fpr[: i + 1], torch.tensor([f_lim], device=device)]) - pro = torch.cat([pro[: i + 1], torch.tensor([p_lim], device=device)]) - else: - fpr = fpr[: i + 1] - pro = pro[: i + 1] - else: - fpr = torch.tensor([0.0, f_lim], device=device) - pro = torch.tensor([0.0, 0.0], device=device) - - return fpr, pro - - def compute(self) -> torch.Tensor: - if not self._preds: - return torch.tensor(0.0) - - cca = self.perform_cca() - preds = torch.cat(self._preds, dim=0) - - fpr, pro = self.compute_pro(cca=cca, preds=preds) - area = self._auc_trapz(fpr, pro) - - denom = fpr[-1].clamp_min(1e-12) # usually == fpr_limit - return area / denom - - -# ----------------------------- -# Utilities: metrics (no sklearn) -# ----------------------------- -def confusion_counts(y_true: np.ndarray, y_pred: np.ndarray): - y_true = y_true.astype(np.int64) - y_pred = y_pred.astype(np.int64) - tp = int(((y_true == 1) & (y_pred == 1)).sum()) - tn = int(((y_true == 0) & (y_pred == 0)).sum()) - fp = int(((y_true == 0) & (y_pred == 1)).sum()) - fn = int(((y_true == 1) & (y_pred == 0)).sum()) - return tp, tn, fp, fn - - -def precision_from_counts(tp: int, fp: int) -> float: - return float(tp / (tp + fp + 1e-12)) - - -def accuracy_from_counts(tp: int, tn: int, fp: int, fn: int) -> float: - return float((tp + tn) / (tp + tn + fp + fn + 1e-12)) - - -def auroc_from_scores(y_true: np.ndarray, y_score: np.ndarray) -> float: - """AUROC via Mann–Whitney U (rank-based), tie-safe.""" - y_true = y_true.astype(np.int64) - y_score = y_score.astype(np.float64) - - pos = (y_true == 1) - neg = (y_true == 0) - n_pos = int(pos.sum()) - n_neg = int(neg.sum()) - if n_pos == 0 or n_neg == 0: - return float("nan") - - order = np.argsort(y_score, kind="mergesort") - ranks = np.empty_like(order, dtype=np.float64) - ranks[order] = np.arange(1, len(y_score) + 1, dtype=np.float64) - - sorted_scores = y_score[order] - i = 0 - while i < len(sorted_scores): - j = i - while j + 1 < len(sorted_scores) and sorted_scores[j + 1] == sorted_scores[i]: - j += 1 - if j > i: - avg_rank = (i + 1 + j + 1) / 2.0 - ranks[order[i : j + 1]] = avg_rank - i = j + 1 - - sum_ranks_pos = float(ranks[pos].sum()) - u = sum_ranks_pos - n_pos * (n_pos + 1) / 2.0 - auc = u / (n_pos * n_neg) - return float(auc) - - -def load_calibration(checkpoint_path: str, calibration_path: Optional[str]) -> dict: - candidates = [] - if calibration_path: - candidates.append(Path(calibration_path)) - checkpoint = Path(checkpoint_path) - candidates.extend( - [checkpoint.parent / "calibration.json", checkpoint.parent.parent / "calibration.json"] - ) - for candidate in candidates: - if candidate.is_file(): - with candidate.open("r", encoding="utf-8") as f: - calibration = json.load(f) - if "image_threshold" not in calibration or "pixel_threshold" not in calibration: - raise ValueError(f"Invalid calibration file: {candidate}") - print(f"Loaded calibration: {candidate}") - return calibration - raise FileNotFoundError( - "No calibration.json found. Run train.py with the updated pipeline or pass --calibration_path." - ) - - -# ----------------------------- -# Data helpers -# ----------------------------- -def iter_images_recursive(root: Path): - exts = {".png", ".jpg", ".jpeg", ".bmp", ".tif", ".tiff"} - for p in root.rglob("*"): - if not p.is_file(): - continue - if p.suffix.lower() not in exts: - continue - if "ground_truth" in {x.lower() for x in p.parts}: - continue - yield p - - -def infer_gt_label_from_path(p: Path) -> Optional[int]: - parts = [x.lower() for x in p.parts] - if "test" not in parts: - return None - i = parts.index("test") - if i + 1 >= len(parts): - return None - return 0 if parts[i + 1] == "good" else 1 - - -def defect_name_from_path(p: Path) -> str: - parts = list(p.parts) - lower = [x.lower() for x in parts] - if "test" in lower: - i = lower.index("test") - if i + 1 < len(parts): - return parts[i + 1] - return p.parent.name - - -def load_gt_mask_mvtec(img_path: Path) -> Optional[np.ndarray]: - parts = list(img_path.parts) - lower = [p.lower() for p in parts] - if "test" not in lower: - return None - ti = lower.index("test") - if ti + 1 >= len(parts): - return None - - defect = parts[ti + 1] - if defect.lower() == "good": - return None - - cat_dir = img_path.parents[2] # .../ - gt_path = cat_dir / "ground_truth" / defect / f"{img_path.stem}_mask.png" - if not gt_path.exists(): - gt_path = cat_dir / "ground_truth" / defect / f"{img_path.stem}.png" - if not gt_path.exists(): - return None - - m = cv2.imread(str(gt_path), cv2.IMREAD_GRAYSCALE) - if m is None: - return None - return ((m > 0).astype(np.uint8) * 255) - - -# ----------------------------- -# Core OOP runner (SSN) -# ----------------------------- -@dataclass -class Sample: - path: Path - gt_label: Optional[int] - score_raw: float # SSN pred_score (sigmoid) by default - amap_hw: np.ndarray # SSN pred_map in crop-space (h,w), float32 [0,1] - orig_rgb: np.ndarray - gt_mask_orig: Optional[np.ndarray] - - class SSNInferenceEngine: def __init__( self, diff --git a/tests/test_regressions.py b/tests/test_regressions.py index ad4e841..b92fdda 100644 --- a/tests/test_regressions.py +++ b/tests/test_regressions.py @@ -1,8 +1,18 @@ +import json + import numpy as np +from PIL import Image import pytest import torch from inference import auroc_from_scores, infer_gt_label_from_path +from anomaly_detection.inference_utils import ( + SelfAUPRO, + defect_name_from_path, + iter_images_recursive, + load_calibration, + load_gt_mask_mvtec, +) from anomaly_detection.modeling import FastFlowModel, SuperSimpleNetModel from anomaly_detection.training import FastFlowEvaluator, SuperSimpleNetEvaluator from ssn_inference import infer_gt_label_from_path as infer_ssn_gt_label @@ -14,6 +24,69 @@ def test_auroc_does_not_silently_reverse_scores(): assert auroc_from_scores(labels, reversed_scores) == pytest.approx(0.0) +@pytest.mark.parametrize("prediction,expected", [ + ("perfect", 1.0), ("reversed", 0.0), ("tied", 0.15), ("one_region", 0.575), +]) +def test_aupro_ranking_ties_and_equal_region_weighting(prediction, expected): + masks = torch.zeros(2, 1, 3, 3) + masks[0, 0, 0, 0] = 1 + masks[1, 0, :2, :2] = 1 # A larger region in a separate image. + if prediction == "perfect": + scores = masks.clone() + elif prediction == "reversed": + scores = 1 - masks + elif prediction == "tied": + scores = torch.full_like(masks, 0.5) + else: + scores = torch.zeros_like(masks) + scores[0] = masks[0] + metric = SelfAUPRO(fpr_limit=0.3) + # Separate updates and both supported binary-mask encodings. + metric.update(scores[:1], masks[:1]) + metric.update(scores[1:], masks[1:] * 255) + # With one region detected, PRO runs from 0.5 to 0.65 over FPR [0, 0.3]. + assert metric.compute().item() == pytest.approx(expected) + + +def test_calibration_lookup_priority_and_missing_or_invalid_files(tmp_path): + checkpoint = tmp_path / "run/checkpoints/best_model.pth" + checkpoint.parent.mkdir(parents=True) + candidates = [ + tmp_path / "explicit.json", + checkpoint.parent / "calibration.json", + checkpoint.parent.parent / "calibration.json", + ] + for index, path in enumerate(candidates): + path.write_text(json.dumps({"image_threshold": index, "pixel_threshold": index + 0.5})) + assert load_calibration(str(checkpoint), str(candidates[0]))["image_threshold"] == 0 + assert load_calibration(str(checkpoint), None)["image_threshold"] == 1 + candidates[1].unlink() + assert load_calibration(str(checkpoint), None)["image_threshold"] == 2 + candidates[2].unlink() + with pytest.raises(FileNotFoundError, match="No calibration.json"): + load_calibration(str(checkpoint), None) + candidates[0].write_text('{"image_threshold": 0.5}') + with pytest.raises(ValueError, match="Invalid calibration file"): + load_calibration(str(checkpoint), str(candidates[0])) + + +@pytest.mark.parametrize("mask_name", ["001_mask.png", "001.png"]) +def test_mvtec_file_helpers_skip_masks_and_find_both_mask_names(tmp_path, mask_name): + good = tmp_path / "bottle/test/good/001.PNG" + defect = tmp_path / "bottle/test/crack/001.png" + mask = tmp_path / "bottle/ground_truth/crack" / mask_name + for path in (good, defect, mask): + path.parent.mkdir(parents=True, exist_ok=True) + Image.fromarray(np.array([[0, 1], [255, 0]], dtype=np.uint8)).save(path) + (tmp_path / "notes.txt").write_text("not an image") + assert set(iter_images_recursive(tmp_path)) == {good, defect} + assert infer_gt_label_from_path(good) == 0 + assert infer_gt_label_from_path(defect) == 1 + assert defect_name_from_path(defect) == "crack" + assert load_gt_mask_mvtec(good) is None + np.testing.assert_array_equal(load_gt_mask_mvtec(defect), [[0, 255], [255, 0]]) + + def test_fastflow_calibration_uses_normal_validation_samples_only(): evaluator = object.__new__(FastFlowEvaluator) predictions = { diff --git a/tests/test_yaml_training.py b/tests/test_yaml_training.py index 5f0be4a..9588bda 100644 --- a/tests/test_yaml_training.py +++ b/tests/test_yaml_training.py @@ -14,8 +14,8 @@ from anomaly_detection.config import load_config, resolve_config from anomaly_detection.pipeline import AnomalyPipeline from anomaly_detection.training.losses import FastFlowNLLLoss, SSNBCELoss, SSNLoss -from inference import FastFlowInferenceEngine -from ssn_inference import SSNInferenceEngine +from inference import FastFlowInferenceEngine, MVTecFastFlowEvaluator, iter_images_recursive, load_calibration +from ssn_inference import SSNInferenceEngine, MVTecSSNEvaluator @pytest.fixture(autouse=True) @@ -169,6 +169,23 @@ def capture_loss(cfg): assert np.isfinite(score) and np.isfinite(anomaly_map).all() assert anomaly_map.shape == (32, 32) + # Exercise the inference scripts' shared file/calibration/metric helpers + # with real predictions from the newly trained checkpoint. + evaluator_cls = MVTecFastFlowEvaluator if model_name == "fastflow" else MVTecSSNEvaluator + evaluator = evaluator_cls( + engine=engine, out_dir=tmp_path / "inference", category="bottle", backbone="resnet18", + min_area=0, thickness=1, fpr_limit=0.3, aupro_downsample=1, save_mode="pred", + calibration=load_calibration(str(checkpoint_path), None), + ) + evaluator.collect(list(iter_images_recursive(tmp_path / "data/bottle/test"))) + labels, _, _, _ = evaluator.compute_image_metrics() + assert sorted(labels.tolist()) == [0, 1] + evaluator.compute_pixel_metrics() + assert np.isfinite(evaluator.image_metrics["auroc"]) + assert np.isfinite(evaluator.pixel_metrics["pixel_auroc_sampled"]) + assert 0 <= evaluator.pixel_metrics["pixel_aupro"] <= 1 + assert evaluator.image_metrics["threshold"] == calibration["image_threshold"] + # A second invocation must not overwrite a completed experiment implicitly. with pytest.raises(FileExistsError, match="Output directory is not empty"): main(["--config", str(path)])